diff options
Diffstat (limited to 'src/zenserver/storage')
25 files changed, 14369 insertions, 0 deletions
diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp new file mode 100644 index 000000000..4803063d7 --- /dev/null +++ b/src/zenserver/storage/admin/admin.cpp @@ -0,0 +1,804 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "admin.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/jobqueue.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zenstore/cache/structuredcachestore.h> +#include <zenstore/gc.h> +#include <zenutil/workerpools.h> + +#if ZEN_WITH_TRACE +# include <zencore/trace.h> +#endif // ZEN_WITH_TRACE + +#if ZEN_USE_MIMALLOC +# include <mimalloc.h> +#endif + +#include "config/config.h" + +#include <chrono> + +namespace zen { + +struct DirStats +{ + uint64_t FileCount = 0; + uint64_t DirCount = 0; + uint64_t ByteCount = 0; +}; + +DirStats +GetStatsForDirectory(std::filesystem::path Dir) +{ + if (!IsDir(Dir)) + return {}; + + struct StatsTraversal : public GetDirectoryContentVisitor + { + virtual void AsyncVisitDirectory(const std::filesystem::path& RelativeRoot, DirectoryContent&& Content) override + { + ZEN_UNUSED(RelativeRoot); + + uint64_t FileCount = Content.FileNames.size(); + uint64_t DirCount = Content.DirectoryNames.size(); + uint64_t FilesSize = 0; + for (uint64_t FileSize : Content.FileSizes) + { + FilesSize += FileSize; + } + TotalBytes += FilesSize; + TotalFileCount += FileCount; + TotalDirCount += DirCount; + } + + std::atomic_uint64_t TotalBytes = 0; + std::atomic_uint64_t TotalFileCount = 0; + std::atomic_uint64_t TotalDirCount = 0; + + DirStats GetStats() + { + return {.FileCount = TotalFileCount.load(), .DirCount = TotalDirCount.load(), .ByteCount = TotalBytes.load()}; + } + } DirTraverser; + + Latch PendingWorkCount(1); + + GetDirectoryContent(Dir, + DirectoryContentFlags::IncludeAllEntries | DirectoryContentFlags::IncludeFileSizes, + DirTraverser, + GetSmallWorkerPool(EWorkloadType::Background), + PendingWorkCount); + PendingWorkCount.CountDown(); + PendingWorkCount.Wait(); + + return DirTraverser.GetStats(); +} + +struct StateDiskStats +{ + DirStats CacheStats; + DirStats CasStats; + DirStats ProjectStats; +}; + +StateDiskStats +GetStatsForStateDirectory(std::filesystem::path StateDir) +{ + StateDiskStats Stats; + Stats.CacheStats = GetStatsForDirectory(StateDir / "cache"); + Stats.CasStats = GetStatsForDirectory(StateDir / "cas"); + Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects"); + return Stats; +} + +HttpAdminService::HttpAdminService(GcScheduler& Scheduler, + JobQueue& BackgroundJobQueue, + ZenCacheStore* CacheStore, + std::function<void()>&& FlushFunction, + const LogPaths& LogPaths, + const ZenServerOptions& ServerOptions) +: m_GcScheduler(Scheduler) +, m_BackgroundJobQueue(BackgroundJobQueue) +, m_CacheStore(CacheStore) +, m_FlushFunction(std::move(FlushFunction)) +, m_LogPaths(LogPaths) +, m_ServerOptions(ServerOptions) +{ + using namespace std::literals; + + m_Router.RegisterRoute( + "health", + [](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.AddPattern("jobid", "([[:digit:]]+?)"); + + m_Router.RegisterRoute( + "jobs", + [&](HttpRouterRequest& Req) { + std::vector<JobQueue::JobInfo> Jobs = m_BackgroundJobQueue.GetJobs(); + CbObjectWriter Obj; + Obj.BeginArray("jobs"); + for (const auto& Job : Jobs) + { + Obj.BeginObject(); + Obj.AddInteger("Id", Job.Id.Id); + Obj.AddString("Status", JobQueue::ToString(Job.Status)); + Obj.EndObject(); + } + Obj.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{jobid}", + [&](HttpRouterRequest& Req) { + const auto& JobIdString = Req.GetCapture(1); + std::optional<uint64_t> JobIdArg = ParseInt<uint64_t>(JobIdString); + if (!JobIdArg) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + JobId Id{.Id = JobIdArg.value_or(0)}; + if (Id.Id == 0) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + ZenContentType::kText, + fmt::format("Invalid Job Id: {}", Id.Id)); + } + + std::optional<JobQueue::JobDetails> CurrentState = m_BackgroundJobQueue.Get(Id); + if (!CurrentState) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { + if (!State.CurrentOp.empty()) + { + Obj.AddString( + "CurrentOp"sv, + State.CurrentOpDetails.empty() ? State.CurrentOp : fmt::format("{}: {}", State.CurrentOp, State.CurrentOpDetails)); + Obj.AddString("Op"sv, State.CurrentOp); + if (!State.CurrentOpDetails.empty()) + { + Obj.AddString("Details"sv, State.CurrentOpDetails); + } + Obj.AddInteger("TotalCount"sv, gsl::narrow<uint64_t>(State.TotalCount)); + Obj.AddInteger("RemainingCount"sv, gsl::narrow<uint64_t>(State.RemainingCount)); + Obj.AddInteger("CurrentOpPercentComplete"sv, + State.TotalCount > 0 + ? gsl::narrow<uint32_t>((100 * (State.TotalCount - State.RemainingCount)) / State.TotalCount) + : 0); + } + if (!State.Messages.empty()) + { + Obj.BeginArray("Messages"); + for (const std::string& Message : State.Messages) + { + Obj.AddString(Message); + } + Obj.EndArray(); + } + if (!State.AbortReason.empty()) + { + Obj.AddString("AbortReason"sv, State.AbortReason); + } + }; + + auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { + auto Age = End - Start; + auto Milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(Age); + return Milliseconds.count() / 1000.0; + }; + + const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); + + switch (CurrentState->Status) + { + case JobQueue::Status::Queued: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Queued"sv); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, Now)); + Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Running: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Running"sv); + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, Now)); + Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Aborted: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Aborted"sv); + + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); + Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Obj.AddInteger("ReturnCode", CurrentState->ReturnCode); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + case JobQueue::Status::Completed: + { + CbObjectWriter Obj; + Obj.AddString("Name"sv, CurrentState->Name); + Obj.AddString("Status"sv, "Complete"sv); + WriteState(Obj, CurrentState->State); + Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); + Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); + Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + } + break; + } + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{jobid}", + [&](HttpRouterRequest& Req) { + const auto& JobIdString = Req.GetCapture(1); + std::optional<uint64_t> JobIdArg = ParseInt<uint64_t>(JobIdString); + if (!JobIdArg) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + JobId Id{.Id = JobIdArg.value_or(0)}; + if (m_BackgroundJobQueue.CancelJob(Id)) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "gc", + [this](HttpRouterRequest& Req) { + const GcSchedulerState State = m_GcScheduler.GetState(); + + const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams(); + + bool Details = false; + if (auto Param = Params.GetValue("details"); Param == "true") + { + Details = true; + } + + CbObjectWriter Response; + Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv); + Response.BeginObject("Config"); + { + Response << "RootDirectory" << State.Config.RootDirectory.string(); + Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval); + Response << "Interval" << ToTimeSpan(State.Config.Interval); + Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); + Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); + Response << "MaxBuildStoreDuration" << ToTimeSpan(State.Config.MaxBuildStoreDuration); + Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; + Response << "Enabled" << State.Config.Enabled; + Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); + Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); + Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); + Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval); + Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1_Deprecated) ? "1" : "2"); + Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent; + Response << "Verbose" << State.Config.Verbose; + Response << "SingleThreaded" << State.Config.SingleThreaded; + Response << "AttachmentPassCount" << State.Config.AttachmentPassCount; + } + Response.EndObject(); + Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; + Response << "HasDiskReserve" << State.HasDiskReserve; + Response << "DiskSize" << NiceBytes(State.DiskSize); + Response << "DiskUsed" << NiceBytes(State.DiskUsed); + Response << "DiskFree" << NiceBytes(State.DiskFree); + + Response.BeginObject("FullGC"); + { + Response << "LastTime" << ToDateTime(State.LastFullGcTime); + Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc); + if (State.Config.DiskSizeSoftLimit != 0) + { + Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); + } + if (State.LastFullGCV2Result) + { + const bool HumanReadable = true; + WriteGCResult(Response, State.LastFullGCV2Result.value(), HumanReadable, Details); + } + else + { + Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration); + Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); + } + if (State.LastFullAttachmentRangeMin != IoHash::Zero || State.LastFullAttachmentRangeMax != IoHash::Max) + { + Response << "AttachmentRangeMin" << State.LastFullAttachmentRangeMin; + Response << "AttachmentRangeMax" << State.LastFullAttachmentRangeMax; + } + } + Response.EndObject(); + Response.BeginObject("LightweightGC"); + { + Response << "LastTime" << ToDateTime(State.LastLightweightGcTime); + Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc); + + if (State.LastLightweightGCV2Result) + { + const bool HumanReadable = true; + WriteGCResult(Response, State.LastLightweightGCV2Result.value(), HumanReadable, Details); + } + else + { + Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration); + Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); + Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); + } + } + Response.EndObject(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "gc", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + GcScheduler::TriggerGcParams GcParams; + + if (auto Param = Params.GetValue("smallobjects"); Param.empty() == false) + { + GcParams.CollectSmallObjects = Param == "true"sv; + } + + if (auto Param = Params.GetValue("maxcacheduration"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.MaxCacheDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("maxbuildstoreduration"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.MaxBuildStoreDuration = std::chrono::seconds(Value.value()); + } + } + + if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + GcParams.DiskSizeSoftLimit = Value.value(); + } + } + + if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) + { + GcParams.SkipCid = Param == "true"sv; + } + + if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) + { + GcParams.SkipDelete = Param == "true"sv; + } + + if (auto Param = Params.GetValue("forceusegcv1"); Param.empty() == false) + { + GcParams.ForceGCVersion = GcVersion::kV1_Deprecated; + } + + if (auto Param = Params.GetValue("forceusegcv2"); Param.empty() == false) + { + GcParams.ForceGCVersion = GcVersion::kV2; + } + + if (auto Param = Params.GetValue("compactblockthreshold"); Param.empty() == false) + { + if (auto Value = ParseInt<uint32_t>(Param)) + { + GcParams.CompactBlockUsageThresholdPercent = Value.value(); + } + } + + if (auto Param = Params.GetValue("verbose"); Param.empty() == false) + { + GcParams.Verbose = Param == "true"sv; + } + + if (auto Param = Params.GetValue("singlethreaded"); Param.empty() == false) + { + GcParams.SingleThreaded = Param == "true"sv; + } + + if (auto Param = Params.GetValue("referencehashlow"); Param.empty() == false) + { + GcParams.AttachmentRangeMin = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("referencehashhigh"); Param.empty() == false) + { + GcParams.AttachmentRangeMax = IoHash::FromHexString(Param); + } + + if (auto Param = Params.GetValue("storecacheattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreCacheAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("storeprojectattachmentmetadata"); Param.empty() == false) + { + GcParams.StoreProjectAttachmentMetaData = Param == "true"sv; + } + + if (auto Param = Params.GetValue("enablevalidation"); Param.empty() == false) + { + GcParams.EnableValidation = Param == "true"sv; + } + + const bool Started = m_GcScheduler.TriggerGc(GcParams); + + CbObjectWriter Response; + Response << "Status"sv << (Started ? "Started"sv : "Running"sv); + HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "gc-stop", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + if (m_GcScheduler.CancelGC()) + { + return HttpReq.WriteResponse(HttpResponseCode::Accepted); + } + HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); + +#if ZEN_USE_MIMALLOC + m_Router.RegisterRoute( + "mi_collect", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + bool Force = false; + + if (auto Param = Params.GetValue("force"); Param.empty() == false) + { + Force = (Param == "true"sv); + } + + ExtendableStringBuilder<256> MiStats; + ExtendableStringBuilder<256> MiStatsAfter; + + auto MiOutputFun = [](const char* msg, void* arg) { + StringBuilderBase* StarsSb = reinterpret_cast<StringBuilderBase*>(arg); + StarsSb->AppendAscii(msg); + }; + + mi_stats_print_out(MiOutputFun, static_cast<StringBuilderBase*>(&MiStats)); + mi_collect(Force); + mi_stats_print_out(MiOutputFun, static_cast<StringBuilderBase*>(&MiStatsAfter)); + + CbObjectWriter Response; + Response << "force"sv << Force; + Response << "stats_before"sv << MiStats; + Response << "stats_after"sv << MiStatsAfter; + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kPost); +#endif + + m_Router.RegisterRoute( + "scrub", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + GcScheduler::TriggerScrubParams ScrubParams; + ScrubParams.MaxTimeslice = std::chrono::seconds(100); + + if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) + { + ScrubParams.SkipDelete = (Param == "true"sv); + } + + if (auto Param = Params.GetValue("skipgc"); Param.empty() == false) + { + ScrubParams.SkipGc = (Param == "true"sv); + } + + if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) + { + ScrubParams.SkipCas = (Param == "true"sv); + } + + m_GcScheduler.TriggerScrub(ScrubParams); + + CbObjectWriter Response; + Response << "ok"sv << true; + Response << "skip_delete" << ScrubParams.SkipDelete; + Response << "skip_gc" << ScrubParams.SkipGc; + Response << "skip_cas" << ScrubParams.SkipCas; + Response << "max_time" << TimeSpan(0, 0, gsl::narrow<int>(ScrubParams.MaxTimeslice.count())); + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "", + [](HttpRouterRequest& Req) { + CbObject Payload = Req.ServerRequest().ReadPayloadObject(); + + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kPost); + +#if ZEN_WITH_TRACE + m_Router.RegisterRoute( + "trace", + [this](HttpRouterRequest& Req) { + bool Enabled = IsTracing(); + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "trace/start", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + TraceOptions TraceOptions; + + if (!IsTracing()) + { + TraceInit("zenserver"); + } + + if (auto Channels = Params.GetValue("channels"); Channels.empty() == false) + { + TraceOptions.Channels = Channels; + } + + if (auto File = Params.GetValue("file"); File.empty() == false) + { + TraceOptions.File = File; + } + else if (auto Host = Params.GetValue("host"); Host.empty() == false) + { + TraceOptions.Host = Host; + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid trace type, use `file` or `host`"sv); + } + + TraceConfigure(TraceOptions); + + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "trace/stop", + [this](HttpRouterRequest& Req) { + if (!IsTracing()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); + } + if (TraceStop()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); + } + else + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + "Failed stopping trace"); + } + }, + HttpVerb::kPost); +#endif // ZEN_WITH_TRACE + + m_Router.RegisterRoute( + "info", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + + Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring(); + Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring(); + + Obj.BeginObject("primary"); + Obj << "data" << m_ServerOptions.DataDir.generic_wstring(); + + try + { + auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir); + + auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) { + Obj.BeginObject(Tag); + Obj << "bytes" << Stats.ByteCount; + Obj << "files" << Stats.FileCount; + Obj << "dirs" << Stats.DirCount; + Obj.EndObject(); + }; + + EmitStats("cache", Stats.CacheStats); + EmitStats("cas", Stats.CasStats); + EmitStats("project", Stats.ProjectStats); + } + catch (const std::exception& Ex) + { + ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what()); + } + Obj.EndObject(); + + try + { + std::vector<CbObject> Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); + + Obj.BeginArray("known"); + + for (const auto& Manifest : Manifests) + { + Obj.AddObject(Manifest); + } + + Obj.EndArray(); + } + catch (const std::exception& Ex) + { + ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what()); + } + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + auto LogLevel = logging::level::ToStringView(logging::GetLogLevel()); + Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); + Obj.BeginObject("cache"); + if (m_CacheStore) + { + const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration(); + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); + Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); + Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); + } + Obj.EndObject(); + Obj.BeginObject("http"); + { + Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); + } + Obj.EndObject(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "logs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool SetCacheLogConfig = false; + ExtendableStringBuilder<256> StringBuilder; + if (m_CacheStore) + { + ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging; + if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) + { + LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) + { + LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; + SetCacheLogConfig = true; + } + if (SetCacheLogConfig) + { + m_CacheStore->SetLoggingConfig(LoggingConfig); + StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", + LoggingConfig.EnableWriteLog ? "true" : "false", + LoggingConfig.EnableAccessLog ? "true" : "false")); + } + } + if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) + { + logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param); + std::string_view LogLevel = logging::level::ToStringView(NewLevel); + if (LogLevel != Param) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid log level '{}'", Param)); + } + logging::SetLogLevel(NewLevel); + if (StringBuilder.Size() > 0) + { + StringBuilder.Append(", "); + } + StringBuilder.Append("loglevel: "); + StringBuilder.Append(Param); + } + return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); + }, + HttpVerb::kPost); + m_Router.RegisterRoute( + "flush", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + m_FlushFunction(); + HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); +} + +HttpAdminService::~HttpAdminService() +{ +} + +const char* +HttpAdminService::BaseUri() const +{ + return "/admin/"; +} + +void +HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/src/zenserver/storage/admin/admin.h b/src/zenserver/storage/admin/admin.h new file mode 100644 index 000000000..9a49f5120 --- /dev/null +++ b/src/zenserver/storage/admin/admin.h @@ -0,0 +1,46 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/compactbinary.h> +#include <zenhttp/httpserver.h> +#include <functional> + +namespace zen { + +class GcScheduler; +class JobQueue; +class ZenCacheStore; +struct ZenServerOptions; + +class HttpAdminService : public zen::HttpService +{ +public: + struct LogPaths + { + std::filesystem::path AbsLogPath; + std::filesystem::path HttpLogPath; + std::filesystem::path CacheLogPath; + }; + HttpAdminService(GcScheduler& Scheduler, + JobQueue& BackgroundJobQueue, + ZenCacheStore* CacheStore, + std::function<void()>&& FlushFunction, + const LogPaths& LogPaths, + const ZenServerOptions& ServerOptions); + ~HttpAdminService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + HttpRequestRouter m_Router; + GcScheduler& m_GcScheduler; + JobQueue& m_BackgroundJobQueue; + ZenCacheStore* m_CacheStore; + std::function<void()> m_FlushFunction; + LogPaths m_LogPaths; + const ZenServerOptions& m_ServerOptions; +}; + +} // namespace zen diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp new file mode 100644 index 000000000..bce993f17 --- /dev/null +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -0,0 +1,561 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpbuildstore.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenhttp/packageformat.h> +#include <zenstore/buildstore/buildstore.h> +#include <zenutil/workerpools.h> + +#include <numeric> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogBuilds, "builds"sv); + +HttpBuildStoreService::HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store) +: m_Log(logging::Get("builds")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_BuildStore(Store) +{ + Initialize(); + + m_StatusService.RegisterHandler("builds", *this); + m_StatsService.RegisterHandler("builds", *this); +} + +HttpBuildStoreService::~HttpBuildStoreService() +{ + m_StatsService.UnregisterHandler("builds", *this); + m_StatusService.UnregisterHandler("builds", *this); +} + +const char* +HttpBuildStoreService::BaseUri() const +{ + return "/builds/"; +} + +void +HttpBuildStoreService::Initialize() +{ + ZEN_LOG_INFO(LogBuilds, "Initializing Builds Service"); + + m_Router.AddPattern("namespace", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("bucket", "([[:alnum:]\\-_.]+)"); + m_Router.AddPattern("buildid", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { PutBlobRequest(Req); }, + HttpVerb::kPut); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/{hash}", + [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", + [this](HttpRouterRequest& Req) { PutMetadataRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/getBlobMetadata", + [this](HttpRouterRequest& Req) { GetMetadatasRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{namespace}/{bucket}/{buildid}/blobs/exists", + [this](HttpRouterRequest& Req) { BlobsExistsRequest(Req); }, + HttpVerb::kPost); +} + +void +HttpBuildStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::HandleRequest"); + metrics::OperationTiming::Scope $(m_HttpRequests); + + m_BuildStoreStats.RequestCount++; + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogBuilds, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpBuildStoreService::PutBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const std::string_view Namespace = Req.GetCapture(1); + const std::string_view Bucket = Req.GetCapture(2); + const std::string_view BuildId = Req.GetCapture(3); + const std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + m_BuildStoreStats.BlobWriteCount++; + IoBuffer Payload = ServerRequest.ReadPayload(); + if (!Payload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} is empty", Hash)); + } + if (Payload.GetContentType() != HttpContentType::kCompressedBinary) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Payload blob {} content type {} is invalid", Hash, ToString(Payload.GetContentType()))); + } + m_BuildStore.PutBlob(BlobHash, ServerRequest.ReadPayload()); + // ZEN_INFO("Stored blob {}. Size: {}", BlobHash, ServerRequest.ReadPayload().GetSize()); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetBlobRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + std::string_view Hash = Req.GetCapture(4); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoHash BlobHash; + if (!IoHash::TryParse(Hash, BlobHash)) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid blob hash '{}'", Hash)); + } + zen::HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (Ranges.size() > 1) + { + // Only a single range is supported + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Multiple ranges in blob request is not supported"); + } + + m_BuildStoreStats.BlobReadCount++; + IoBuffer Blob = m_BuildStore.GetBlob(BlobHash); + if (!Blob) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Blob with hash '{}' could not be found", Hash)); + } + // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); + m_BuildStoreStats.BlobHitCount++; + if (HasRange) + { + const HttpRange& Range = Ranges.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); + if (Range.Start + RangeSize > BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, Range.Start, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } +} + +void +HttpBuildStoreService::PutMetadataRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::PutMetadataRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + + IoBuffer MetaPayload = ServerRequest.ReadPayload(); + if (MetaPayload.GetContentType() != ZenContentType::kCbPackage) + { + throw std::runtime_error(fmt::format("PutMetadataRequest payload has unexpected payload type '{}', expected '{}'", + ToString(MetaPayload.GetContentType()), + ToString(ZenContentType::kCbPackage))); + } + CbPackage Message = ParsePackageMessage(MetaPayload); + + CbObjectView MessageObject = Message.GetObject(); + if (!MessageObject) + { + throw std::runtime_error("PutMetadataRequest payload object is missing"); + } + CbArrayView BlobsArray = MessageObject["blobHashes"sv].AsArrayView(); + CbArrayView MetadataArray = MessageObject["metadatas"sv].AsArrayView(); + + const uint64_t BlobCount = BlobsArray.Num(); + if (BlobCount == 0) + { + throw std::runtime_error("PutMetadataRequest blobs array is empty"); + } + if (BlobCount != MetadataArray.Num()) + { + throw std::runtime_error( + fmt::format("PutMetadataRequest metadata array size {} does not match blobs array size {}", MetadataArray.Num(), BlobCount)); + } + + std::vector<IoHash> BlobHashes; + std::vector<IoBuffer> MetadataPayloads; + + BlobHashes.reserve(BlobCount); + MetadataPayloads.reserve(BlobCount); + + auto BlobsArrayIt = begin(BlobsArray); + auto MetadataArrayIt = begin(MetadataArray); + while (BlobsArrayIt != end(BlobsArray)) + { + const IoHash BlobHash = (*BlobsArrayIt).AsHash(); + const IoHash MetadataHash = (*MetadataArrayIt).AsAttachment(); + + const CbAttachment* Attachment = Message.FindAttachment(MetadataHash); + if (Attachment == nullptr) + { + throw std::runtime_error(fmt::format("Blob metadata attachment {} is missing", MetadataHash)); + } + BlobHashes.push_back(BlobHash); + if (Attachment->IsObject()) + { + MetadataPayloads.push_back(Attachment->AsObject().GetBuffer().MakeOwned().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCbObject); + } + else if (Attachment->IsCompressedBinary()) + { + MetadataPayloads.push_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kCompressedBinary); + } + else + { + ZEN_ASSERT(Attachment->IsBinary()); + MetadataPayloads.push_back(Attachment->AsBinary().AsIoBuffer()); + MetadataPayloads.back().SetContentType(ZenContentType::kBinary); + } + + BlobsArrayIt++; + MetadataArrayIt++; + } + m_BuildStore.PutMetadatas(BlobHashes, MetadataPayloads, &GetSmallWorkerPool(EWorkloadType::Burst)); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpBuildStoreService::GetMetadatasRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::GetMetadatasRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for metadata request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for metadata request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for metadata request is missing 'blobHashes' array"); + } + const uint64_t BlobCount = BlobsArray.Num(); + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobCount); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for metadata 'blobHashes' array contains invalid field type: {}", Type)); + } + } + m_BuildStoreStats.BlobMetaReadCount += BlobRawHashes.size(); + std::vector<IoBuffer> BlockMetadatas = m_BuildStore.GetMetadatas(BlobRawHashes, &GetSmallWorkerPool(EWorkloadType::Burst)); + + CbPackage ResponsePackage; + std::vector<CbAttachment> Attachments; + tsl::robin_set<IoHash, IoHash::Hasher> AttachmentHashes; + Attachments.reserve(BlobCount); + AttachmentHashes.reserve(BlobCount); + { + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginArray("blobHashes"); + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (BlockMetadatas[BlockHashIndex]) + { + const IoHash& BlockHash = BlobRawHashes[BlockHashIndex]; + ResponseWriter.AddHash(BlockHash); + } + } + ResponseWriter.EndArray(); // blobHashes + + ResponseWriter.BeginArray("metadatas"); + + for (size_t BlockHashIndex = 0; BlockHashIndex < BlobRawHashes.size(); BlockHashIndex++) + { + if (IoBuffer Metadata = BlockMetadatas[BlockHashIndex]; Metadata) + { + switch (Metadata.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbObject Object = CbObject(SharedBuffer(std::move(Metadata)).MakeOwned()); + const IoHash ObjectHash = Object.GetHash(); + ResponseWriter.AddBinaryAttachment(ObjectHash); + if (!AttachmentHashes.contains(ObjectHash)) + { + Attachments.push_back(CbAttachment(Object, ObjectHash)); + AttachmentHashes.insert(ObjectHash); + } + } + break; + case ZenContentType::kCompressedBinary: + { + IoHash RawHash; + uint64_t _; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(std::move(Metadata)), RawHash, _); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(Compressed, RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + default: + { + const IoHash RawHash = IoHash::HashBuffer(Metadata); + ResponseWriter.AddBinaryAttachment(RawHash); + if (!AttachmentHashes.contains(RawHash)) + { + Attachments.push_back(CbAttachment(SharedBuffer(Metadata), RawHash)); + AttachmentHashes.insert(RawHash); + } + } + break; + } + } + } + + ResponseWriter.EndArray(); // metadatas + + ResponsePackage.SetObject(ResponseWriter.Save()); + } + ResponsePackage.AddAttachments(Attachments); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); +} + +void +HttpBuildStoreService::BlobsExistsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::BlobsExistsRequest"); + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Namespace = Req.GetCapture(1); + std::string_view Bucket = Req.GetCapture(2); + std::string_view BuildId = Req.GetCapture(3); + ZEN_UNUSED(Namespace, Bucket, BuildId); + IoBuffer RequestPayload = ServerRequest.ReadPayload(); + if (!RequestPayload) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Expected compact binary body for blob exists request, body is missing"); + } + if (RequestPayload.GetContentType() != HttpContentType::kCbObject) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Expected compact binary body for blob exists request, got {}", ToString(RequestPayload.GetContentType()))); + } + if (CbValidateError ValidateError = ValidateCompactBinary(RequestPayload.GetView(), CbValidateMode::Default); + ValidateError != CbValidateError::None) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request is not valid, reason: {}", ToString(ValidateError))); + } + CbObject RequestObject = LoadCompactBinaryObject(RequestPayload); + CbArrayView BlobsArray = RequestObject["blobHashes"sv].AsArrayView(); + if (!BlobsArray) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Compact binary body for blob exists request is missing 'blobHashes' array"); + } + + std::vector<IoHash> BlobRawHashes; + BlobRawHashes.reserve(BlobsArray.Num()); + for (CbFieldView BlockHashView : BlobsArray) + { + BlobRawHashes.push_back(BlockHashView.AsHash()); + if (BlobRawHashes.back() == IoHash::Zero) + { + const uint8_t Type = (uint8_t)BlockHashView.GetValue().GetType(); + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Compact binary body for blob exists request 'blobHashes' array contains invalid field type: {}", Type)); + } + } + + m_BuildStoreStats.BlobExistsCount += BlobRawHashes.size(); + std::vector<BuildStore::BlobExistsResult> BlobsExists = m_BuildStore.BlobsExists(BlobRawHashes); + CbObjectWriter ResponseWriter(9 * BlobsExists.size()); + ResponseWriter.BeginArray("blobExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasBody); + if (BlobExists.HasBody) + { + m_BuildStoreStats.BlobExistsBodyHitCount++; + } + } + ResponseWriter.EndArray(); // blobExist + ResponseWriter.BeginArray("metadataExists"sv); + for (const BuildStore::BlobExistsResult& BlobExists : BlobsExists) + { + ResponseWriter.AddBool(BlobExists.HasMetadata); + if (BlobExists.HasMetadata) + { + m_BuildStoreStats.BlobExistsMetaHitCount++; + } + } + ResponseWriter.EndArray(); // metadataExists + CbObject ResponseObject = ResponseWriter.Save(); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ResponseObject); +} + +void +HttpBuildStoreService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("builds"); + { + Cbo.BeginObject("blobs"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobReadCount << "writecount" << m_BuildStoreStats.BlobWriteCount << "hitcount" + << m_BuildStoreStats.BlobHitCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("metadata"); + { + Cbo << "readcount" << m_BuildStoreStats.BlobMetaReadCount << "writecount" << m_BuildStoreStats.BlobMetaWriteCount << "hitcount" + << m_BuildStoreStats.BlobMetaHitCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_BuildStoreStats.RequestCount; + Cbo << "badrequestcount" << m_BuildStoreStats.BadRequestCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + BuildStore::StorageStats StorageStats = m_BuildStore.GetStorageStats(); + + Cbo << "count" << StorageStats.EntryCount; + Cbo << "bytes" << StorageStats.BlobBytes + StorageStats.MetadataByteCount; + Cbo.BeginObject("blobs"); + { + Cbo << "count" << StorageStats.BlobCount; + Cbo << "bytes" << StorageStats.BlobBytes; + } + Cbo.EndObject(); // blobs + + Cbo.BeginObject("metadata"); + { + Cbo << "count" << StorageStats.MetadataCount; + Cbo << "bytes" << StorageStats.MetadataByteCount; + } + Cbo.EndObject(); // metadata + } + Cbo.EndObject(); // size + + return Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpBuildStoreService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +} // namespace zen diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h new file mode 100644 index 000000000..50cb5db12 --- /dev/null +++ b/src/zenserver/storage/buildstore/httpbuildstore.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> + +#include <filesystem> + +namespace zen { + +class BuildStore; + +class HttpBuildStoreService final : public zen::HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpBuildStoreService(HttpStatusService& StatusService, HttpStatsService& StatsService, BuildStore& Store); + virtual ~HttpBuildStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct BuildStoreStats + { + std::atomic_uint64_t BlobReadCount{}; + std::atomic_uint64_t BlobHitCount{}; + std::atomic_uint64_t BlobWriteCount{}; + std::atomic_uint64_t BlobMetaReadCount{}; + std::atomic_uint64_t BlobMetaHitCount{}; + std::atomic_uint64_t BlobMetaWriteCount{}; + std::atomic_uint64_t BlobExistsCount{}; + std::atomic_uint64_t BlobExistsBodyHitCount{}; + std::atomic_uint64_t BlobExistsMetaHitCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void Initialize(); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void PutBlobRequest(HttpRouterRequest& Req); + void GetBlobRequest(HttpRouterRequest& Req); + + void PutMetadataRequest(HttpRouterRequest& Req); + void GetMetadatasRequest(HttpRouterRequest& Req); + + void BlobsExistsRequest(HttpRouterRequest& Req); + + HttpRequestRouter m_Router; + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + + BuildStore& m_BuildStore; + BuildStoreStats m_BuildStoreStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp new file mode 100644 index 000000000..ece1d7a46 --- /dev/null +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -0,0 +1,2052 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpstructuredcache.h" + +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/compress.h> +#include <zencore/enumflags.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/memory/llm.h> +#include <zencore/parallelwork.h> +#include <zencore/scopeguard.h> +#include <zencore/stream.h> +#include <zencore/timer.h> +#include <zencore/trace.h> +#include <zencore/workthreadpool.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/packageformat.h> +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenstore/cache/cache.h> +#include <zenstore/cache/structuredcachestore.h> +#include <zenstore/gc.h> +#include <zenutil/rpcrecording.h> +#include <zenutil/workerpools.h> + +#include "storage/upstream/upstreamcache.h" +#include "storage/upstream/zen.h" +#include "zenstore/cidstore.h" +#include "zenstore/scrubcontext.h" + +#include <algorithm> +#include <atomic> +#include <filesystem> +#include <queue> +#include <thread> + +#include <gsl/gsl-lite.hpp> + +namespace zen { + +const FLLMTag& +GetCacheHttpTag() +{ + static FLLMTag CacheHttpTag("http", FLLMTag("cache")); + + return CacheHttpTag; +} + +extern const FLLMTag& GetCacheRpcTag(); + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +CachePolicy +ParseCachePolicy(const HttpServerRequest::QueryParams& QueryParams) +{ + std::string_view PolicyText = QueryParams.GetValue("Policy"sv); + return !PolicyText.empty() ? zen::ParseCachePolicy(PolicyText) : CachePolicy::Default; +} + +namespace { + static constinit std::string_view HttpZCacheRPCPrefix = "$rpc"sv; + static constinit std::string_view HttpZCacheUtilStartRecording = "exec$/start-recording"sv; + static constinit std::string_view HttpZCacheUtilStopRecording = "exec$/stop-recording"sv; + static constinit std::string_view HttpZCacheUtilReplayRecording = "exec$/replay-recording"sv; + static constinit std::string_view HttpZCacheDetailsPrefix = "details$"sv; +} // namespace + +////////////////////////////////////////////////////////////////////////// + +HttpStructuredCacheService::HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache, + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache) +: m_Log(logging::Get("cache")) +, m_CacheStore(InCacheStore) +, m_StatsService(StatsService) +, m_StatusService(StatusService) +, m_CidStore(InCidStore) +, m_UpstreamCache(UpstreamCache) +, m_DiskWriteBlocker(InDiskWriteBlocker) +, m_OpenProcessCache(InOpenProcessCache) +, m_RpcHandler(m_Log, m_CacheStats, UpstreamCache, InCacheStore, InCidStore, InDiskWriteBlocker) +{ + m_StatsService.RegisterHandler("z$", *this); + m_StatusService.RegisterHandler("z$", *this); +} + +HttpStructuredCacheService::~HttpStructuredCacheService() +{ + ZEN_INFO("closing structured cache"); + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + + m_StatsService.UnregisterHandler("z$", *this); + m_StatusService.UnregisterHandler("z$", *this); +} + +const char* +HttpStructuredCacheService::BaseUri() const +{ + return "/z$/"; +} + +void +HttpStructuredCacheService::Flush() +{ + m_CacheStore.Flush(); +} + +void +HttpStructuredCacheService::HandleDetailsRequest(HttpServerRequest& Request) +{ + std::string_view Key = Request.RelativeUri(); + std::vector<std::string> Tokens; + uint32_t TokenCount = ForEachStrTok(Key, '/', [&Tokens](std::string_view Token) { + Tokens.push_back(std::string(Token)); + return true; + }); + std::string FilterNamespace; + std::string FilterBucket; + std::string FilterValue; + switch (TokenCount) + { + case 1: + break; + case 2: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + case 3: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterBucket = Tokens[2]; + if (FilterBucket.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + case 4: + { + FilterNamespace = Tokens[1]; + if (FilterNamespace.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterBucket = Tokens[2]; + if (FilterBucket.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + FilterValue = Tokens[3]; + if (FilterValue.empty()) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + } + break; + default: + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + bool CSV = Params.GetValue("csv") == "true"; + bool Details = Params.GetValue("details") == "true"; + bool AttachmentDetails = Params.GetValue("attachmentdetails") == "true"; + + std::chrono::seconds NowSeconds = std::chrono::duration_cast<std::chrono::seconds>(GcClock::Now().time_since_epoch()); + CacheValueDetails ValueDetails = m_CacheStore.GetValueDetails(FilterNamespace, FilterBucket, FilterValue); + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + if (AttachmentDetails) + { + CSVWriter << "Namespace, Bucket, Key, Cid, Size"; + } + else if (Details) + { + CSVWriter << "Namespace, Bucket, Key, Size, RawSize, RawHash, ContentType, Age, AttachmentsCount, AttachmentsSize"; + } + else + { + CSVWriter << "Namespace, Bucket, Key"; + } + for (const auto& NamespaceIt : ValueDetails.Namespaces) + { + const std::string& Namespace = NamespaceIt.first; + for (const auto& BucketIt : NamespaceIt.second.Buckets) + { + const std::string& Bucket = BucketIt.first; + for (const auto& ValueIt : BucketIt.second.Values) + { + if (AttachmentDetails) + { + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + CSVWriter << "\r\n" + << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << Hash.ToHexString() + << ", " << gsl::narrow<uint64_t>(Payload.GetSize()); + } + } + else if (Details) + { + std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>( + GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); + CSVWriter << "\r\n" + << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString() << ", " << ValueIt.second.Size << "," + << ValueIt.second.RawSize << "," << ValueIt.second.RawHash.ToHexString() << ", " + << ToString(ValueIt.second.ContentType) << ", " << (NowSeconds.count() - LastAccessedSeconds.count()) + << ", " << gsl::narrow<uint64_t>(ValueIt.second.Attachments.size()); + size_t AttachmentsSize = 0; + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + AttachmentsSize += Payload.GetSize(); + } + CSVWriter << ", " << gsl::narrow<uint64_t>(AttachmentsSize); + } + else + { + CSVWriter << "\r\n" << Namespace << "," << Bucket << "," << ValueIt.first.ToHexString(); + } + } + } + } + return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("namespaces"); + { + for (const auto& NamespaceIt : ValueDetails.Namespaces) + { + const std::string& Namespace = NamespaceIt.first; + Cbo.BeginObject(); + { + Cbo.AddString("name", Namespace); + Cbo.BeginArray("buckets"); + { + for (const auto& BucketIt : NamespaceIt.second.Buckets) + { + const std::string& Bucket = BucketIt.first; + Cbo.BeginObject(); + { + Cbo.AddString("name", Bucket); + Cbo.BeginArray("values"); + { + for (const auto& ValueIt : BucketIt.second.Values) + { + std::chrono::seconds LastAccessedSeconds = std::chrono::duration_cast<std::chrono::seconds>( + GcClock::TimePointFromTick(ValueIt.second.LastAccess).time_since_epoch()); + Cbo.BeginObject(); + { + Cbo.AddHash("key", ValueIt.first); + if (Details) + { + Cbo.AddInteger("size", ValueIt.second.Size); + if (ValueIt.second.Size > 0 && ValueIt.second.RawSize != 0 && + ValueIt.second.RawSize != ValueIt.second.Size) + { + Cbo.AddInteger("rawsize", ValueIt.second.RawSize); + Cbo.AddHash("rawhash", ValueIt.second.RawHash); + } + Cbo.AddString("contenttype", ToString(ValueIt.second.ContentType)); + Cbo.AddInteger("age", + gsl::narrow<uint64_t>(NowSeconds.count() - LastAccessedSeconds.count())); + if (ValueIt.second.Attachments.size() > 0) + { + if (AttachmentDetails) + { + Cbo.BeginArray("attachments"); + { + for (const IoHash& Hash : ValueIt.second.Attachments) + { + Cbo.BeginObject(); + Cbo.AddHash("cid", Hash); + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + Cbo.AddInteger("size", gsl::narrow<uint64_t>(Payload.GetSize())); + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + else + { + Cbo.AddInteger("attachmentcount", + gsl::narrow<uint64_t>(ValueIt.second.Attachments.size())); + size_t AttachmentsSize = 0; + for (const IoHash& Hash : ValueIt.second.Attachments) + { + IoBuffer Payload = m_CidStore.FindChunkByCid(Hash); + AttachmentsSize += Payload.GetSize(); + } + Cbo.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize)); + } + } + } + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + } + Cbo.EndArray(); + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpStructuredCacheService::HandleRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("z$::Http::HandleRequest"); + + ZEN_MEMSCOPE(GetCacheHttpTag()); + + metrics::OperationTiming::Scope $(m_HttpRequests); + + const std::string_view Key = Request.RelativeUri(); + + std::string_view UriNamespace; + + if (Key.ends_with(HttpZCacheRPCPrefix)) + { + const size_t RpcOffset = Key.length() - HttpZCacheRPCPrefix.length(); + + if (RpcOffset) + { + std::string_view KeyPrefix = Key.substr(0, RpcOffset); + + if (KeyPrefix.back() == '/') + { + KeyPrefix.remove_suffix(1); + + UriNamespace = KeyPrefix; + } + } + + return HandleRpcRequest(Request, UriNamespace); + } + + if (Key == HttpZCacheUtilStartRecording) + { + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); + + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + + m_RequestRecorder = cache::MakeDiskRequestRecorder(RecordPath); + m_RequestRecordingEnabled.store(true); + } + ZEN_INFO("cache RPC recording STARTED -> '{}'", RecordPath); + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key == HttpZCacheUtilStopRecording) + { + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + ZEN_INFO("cache RPC recording STOPPED"); + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key == HttpZCacheUtilReplayRecording) + { + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + { + RwLock::ExclusiveLockScope _(m_RequestRecordingLock); + m_RequestRecordingEnabled.store(false); + m_RequestRecorder.reset(); + } + + HttpServerRequest::QueryParams Params = Request.GetQueryParams(); + + std::string RecordPath = UrlDecode(Params.GetValue("path")); + + uint32_t ThreadCount = GetHardwareConcurrency(); + if (auto Param = Params.GetValue("thread_count"); Param.empty() == false) + { + if (auto Value = ParseInt<uint64_t>(Param)) + { + ThreadCount = gsl::narrow<uint32_t>(Value.value()); + } + } + + ZEN_INFO("initiating cache RPC replay using {} threads, from '{}'", ThreadCount, RecordPath); + + std::unique_ptr<cache::IRpcRequestReplayer> Replayer(cache::MakeDiskRequestReplayer(RecordPath, false)); + ReplayRequestRecorder(RequestContext, *Replayer, ThreadCount < 1 ? 1 : ThreadCount); + + ZEN_INFO("cache RPC replay COMPLETED"); + + Request.WriteResponse(HttpResponseCode::OK); + return; + } + + if (Key.starts_with(HttpZCacheDetailsPrefix)) + { + HandleDetailsRequest(Request); + return; + } + + HttpCacheRequestData RequestData; + if (!HttpCacheRequestParseRelativeUri(Key, ZenCacheStore::DefaultNamespace, RequestData)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); // invalid URL + } + + if (RequestData.ValueContentId.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + ZEN_ASSERT(RequestData.HashKey.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = RequestData.ValueContentId.value()}; + return HandleCacheChunkRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } + + if (RequestData.HashKey.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + ZEN_ASSERT(RequestData.Bucket.has_value()); + CacheRef Ref = {.Namespace = RequestData.Namespace.value(), + .BucketSegment = RequestData.Bucket.value(), + .HashKey = RequestData.HashKey.value(), + .ValueContentId = IoHash::Zero}; + return HandleCacheRecordRequest(Request, Ref, ParseCachePolicy(Request.GetQueryParams())); + } + + if (RequestData.Bucket.has_value()) + { + ZEN_ASSERT(RequestData.Namespace.has_value()); + return HandleCacheBucketRequest(Request, RequestData.Namespace.value(), RequestData.Bucket.value()); + } + + if (RequestData.Namespace.has_value()) + { + return HandleCacheNamespaceRequest(Request, RequestData.Namespace.value()); + } + return HandleCacheRequest(Request); +} + +void +HttpStructuredCacheService::HandleCacheRequest(HttpServerRequest& Request) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + ZenCacheStore::Info Info = m_CacheStore.GetInfo(); + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("Configuration"); + { + ExtendableStringBuilder<128> BasePathString; + BasePathString << Info.BasePath.u8string(); + ResponseWriter.AddString("BasePath"sv, BasePathString.ToView()); + ResponseWriter.AddBool("AllowAutomaticCreationOfNamespaces", Info.Config.AllowAutomaticCreationOfNamespaces); + ResponseWriter.BeginObject("Logging"); + { + ResponseWriter.AddBool("EnableWriteLog", Info.Config.Logging.EnableWriteLog); + ResponseWriter.AddBool("EnableAccessLog", Info.Config.Logging.EnableAccessLog); + } + ResponseWriter.EndObject(); + } + ResponseWriter.EndObject(); + + std::sort(begin(Info.NamespaceNames), end(Info.NamespaceNames), [](std::string_view L, std::string_view R) { + return L.compare(R) < 0; + }); + ResponseWriter.BeginArray("Namespaces"); + for (const std::string& NamespaceName : Info.NamespaceNames) + { + ResponseWriter.AddString(NamespaceName); + } + ResponseWriter.EndArray(); + ResponseWriter.BeginObject("StorageSize"); + { + ResponseWriter.AddInteger("DiskSize", Info.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize", Info.StorageSize.MemorySize); + } + + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("DiskEntryCount", Info.DiskEntryCount); + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + default: + m_CacheStats.BadRequestCount++; + break; + } +} + +void +HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view NamespaceName) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + std::optional<ZenCacheNamespace::Info> Info = m_CacheStore.GetNamespaceInfo(NamespaceName); + if (!Info.has_value()) + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("Configuration"); + { + ExtendableStringBuilder<128> BasePathString; + BasePathString << Info->RootDir.u8string(); + ResponseWriter.AddString("RootDir"sv, BasePathString.ToView()); + ResponseWriter.AddInteger("MaxBlockSize"sv, Info->Config.DiskLayerConfig.BucketConfig.MaxBlockSize); + ResponseWriter.AddInteger("PayloadAlignment"sv, Info->Config.DiskLayerConfig.BucketConfig.PayloadAlignment); + ResponseWriter.AddInteger("MemCacheSizeThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold); + ResponseWriter.AddInteger("LargeObjectThreshold"sv, Info->Config.DiskLayerConfig.BucketConfig.LargeObjectThreshold); + ResponseWriter.AddInteger("MemCacheTargetFootprintBytes"sv, Info->Config.DiskLayerConfig.MemCacheTargetFootprintBytes); + ResponseWriter.AddInteger("MemCacheTrimIntervalSeconds"sv, Info->Config.DiskLayerConfig.MemCacheTrimIntervalSeconds); + ResponseWriter.AddInteger("MemCacheMaxAgeSeconds"sv, Info->Config.DiskLayerConfig.MemCacheMaxAgeSeconds); + } + ResponseWriter.EndObject(); + + std::sort(begin(Info->BucketNames), end(Info->BucketNames), [](std::string_view L, std::string_view R) { + return L.compare(R) < 0; + }); + + ResponseWriter.BeginArray("Buckets"sv); + for (const std::string& BucketName : Info->BucketNames) + { + ResponseWriter.AddString(BucketName); + } + ResponseWriter.EndArray(); + + ResponseWriter.BeginObject("StorageSize"sv); + { + ResponseWriter.AddInteger("DiskSize"sv, Info->DiskLayerInfo.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize"sv, Info->DiskLayerInfo.StorageSize.MemorySize); + } + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("EntryCount", Info->DiskLayerInfo.EntryCount); + + if (auto Buckets = HttpServerRequest::Decode(Request.GetQueryParams().GetValue("bucketsizes")); !Buckets.empty()) + { + ResponseWriter.BeginObject("BucketSizes"); + + ResponseWriter.BeginArray("Buckets"); + + std::vector<std::string> BucketNames; + if (Buckets == "*") // Get all - empty FieldFilter equal getting all fields + { + BucketNames = Info.value().BucketNames; + } + else + { + ForEachStrTok(Buckets, ',', [&](std::string_view BucketName) { + BucketNames.push_back(std::string(BucketName)); + return true; + }); + } + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + std::vector<IoHash> AllAttachments; + for (const std::string& BucketName : BucketNames) + { + ResponseWriter.BeginObject(); + ResponseWriter << "Name" << BucketName; + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + AllAttachments.insert(AllAttachments.end(), ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + } + ResponseWriter.EndObject(); + } + + ResponseWriter.EndArray(); + + ResponseWriter.BeginObject("Attachments"); + std::sort(AllAttachments.begin(), AllAttachments.end()); + auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end()); + AllAttachments.erase(NewEnd, AllAttachments.end()); + + uint64_t AttachmentsSize = 0; + + m_CidStore.IterateChunks( + AllAttachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "Count" << AllAttachments.size(); + ResponseWriter << "Size" << AttachmentsSize; + + ResponseWriter.EndObject(); + + ResponseWriter.EndObject(); + } + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + + case HttpVerb::kDelete: + // Drop namespace + { + if (m_CacheStore.DropNamespace(NamespaceName)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + } + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, + std::string_view NamespaceName, + std::string_view BucketName) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + { + std::optional<ZenCacheNamespace::BucketInfo> Info = m_CacheStore.GetBucketInfo(NamespaceName, BucketName); + if (!Info.has_value()) + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter ResponseWriter; + + ResponseWriter.BeginObject("StorageSize"); + { + ResponseWriter.AddInteger("DiskSize", Info->DiskLayerInfo.StorageSize.DiskSize); + ResponseWriter.AddInteger("MemorySize", Info->DiskLayerInfo.StorageSize.MemorySize); + } + ResponseWriter.EndObject(); + + ResponseWriter.AddInteger("DiskEntryCount", Info->DiskLayerInfo.EntryCount); + + if (auto GetBucketSize = Request.GetQueryParams().GetValue("bucketsize"); GetBucketSize == "true") + { + CacheContentStats ContentStats; + bool Success = m_CacheStore.GetContentStats(NamespaceName, BucketName, ContentStats); + if (Success) + { + size_t ValuesSize = 0; + for (const uint64_t Size : ContentStats.ValueSizes) + { + ValuesSize += Size; + } + + std::sort(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + auto NewEnd = std::unique(ContentStats.Attachments.begin(), ContentStats.Attachments.end()); + ContentStats.Attachments.erase(NewEnd, ContentStats.Attachments.end()); + + ResponseWriter << "Count" << ContentStats.ValueSizes.size(); + ResponseWriter << "StructuredCount" << ContentStats.StructuredValuesCount; + ResponseWriter << "StandaloneCount" << ContentStats.StandaloneValuesCount; + ResponseWriter << "Size" << ValuesSize; + ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); + + uint64_t AttachmentsSize = 0; + + WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + m_CidStore.IterateChunks( + ContentStats.Attachments, + [&](size_t Index, const IoBuffer& Payload) { + ZEN_UNUSED(Index); + AttachmentsSize += Payload.GetSize(); + return true; + }, + &WorkerPool, + 8u * 1024u); + + ResponseWriter << "AttachmentsSize" << AttachmentsSize; + } + } + + return Request.WriteResponse(HttpResponseCode::OK, ResponseWriter.Save()); + } + break; + + case HttpVerb::kDelete: + // Drop bucket + { + if (m_CacheStore.DropBucket(NamespaceName, BucketName)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + return Request.WriteResponse(HttpResponseCode::NotFound); + } + } + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + HandleGetCacheRecord(Request, Ref, PolicyFromUrl); + break; + + case HttpVerb::kPut: + HandlePutCacheRecord(Request, Ref, PolicyFromUrl); + break; + + default: + break; + } +} + +void +HttpStructuredCacheService::HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + const ZenContentType AcceptType = Request.AcceptContentType(); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + + bool Success = false; + uint32_t MissingCount = 0; + ZenCacheValue ClientResultValue; + if (!EnumHasAnyFlags(PolicyFromUrl, CachePolicy::Query)) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + Stopwatch Timer; + + if (EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal) && + m_CacheStore.Get(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue)) + { + Success = true; + ZenContentType ContentType = ClientResultValue.Value.GetContentType(); + + if (AcceptType == ZenContentType::kCbPackage) + { + if (ContentType == ZenContentType::kCbObject) + { + CbPackage Package; + CbValidateError ValidateError = CbValidateError::None; + if (CbObject PackageObject = ValidateAndReadCompactBinaryObject(std::move(ClientResultValue.Value), ValidateError); + ValidateError == CbValidateError::None) + { + CbObjectView CacheRecord(ClientResultValue.Value.Data()); + CacheRecord.IterateAttachments([this, &MissingCount, &Package, SkipData](CbFieldView AttachmentHash) { + if (SkipData) + { + if (!m_CidStore.ContainsChunk(AttachmentHash.AsHash())) + { + MissingCount++; + } + } + else + { + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(AttachmentHash.AsHash())) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash.AsHash())); + } + else + { + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash.AsHash()); + MissingCount++; + } + } + else + { + MissingCount++; + } + } + }); + + Success = MissingCount == 0 || PartialRecord; + } + else + { + ZEN_WARN("Invalid compact binary payload returned for {}/{}/{} ({}). Reason: '{}'", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(ValidateError)); + Success = false; + } + + if (Success) + { + CbObject PackageObject = LoadCompactBinaryObject(std::move(ClientResultValue.Value)); + + Package.SetObject(std::move(PackageObject)); + + BinaryWriter MemStream; + Package.Save(MemStream); + + ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + ClientResultValue.Value.SetContentType(HttpContentType::kCbPackage); + } + } + else + { + Success = false; + } + } + else if (AcceptType != ClientResultValue.Value.GetContentType() && AcceptType != ZenContentType::kUnknownContentType && + AcceptType != ZenContentType::kBinary) + { + Success = false; + } + } + + if (Success) + { + ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (LOCAL) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(ClientResultValue.Value.Size()), + ToString(ClientResultValue.Value.GetContentType()), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + m_CacheStats.HitCount++; + if (SkipData && AcceptType != ZenContentType::kCbPackage && AcceptType != ZenContentType::kCbObject) + { + return Request.WriteResponse(HttpResponseCode::OK); + } + else + { + // kCbPackage handled SkipData when constructing the ClientResultValue, kcbObject ignores SkipData + return Request.WriteResponse((MissingCount == 0) ? HttpResponseCode::OK : HttpResponseCode::PartialContent, + ClientResultValue.Value.GetContentType(), + ClientResultValue.Value); + } + } + else if (!HasUpstream || !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryRemote)) + { + ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.MissCount++; + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + // Issue upstream query asynchronously in order to keep requests flowing without + // hogging I/O servicing threads with blocking work + + uint64_t LocalElapsedTimeUs = Timer.GetElapsedTimeUs(); + + Request.WriteResponseAsync([this, AcceptType, PolicyFromUrl, Ref, LocalElapsedTimeUs, RequestContext](HttpServerRequest& AsyncRequest) { + Stopwatch Timer; + bool Success = false; + const bool PartialRecord = EnumHasAllFlags(PolicyFromUrl, CachePolicy::PartialRecord); + const bool QueryLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + const bool StoreLocal = EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreLocal) && AreDiskWritesAllowed(); + const bool SkipData = EnumHasAllFlags(PolicyFromUrl, CachePolicy::SkipData); + ZenCacheValue ClientResultValue; + + metrics::OperationTiming::Scope $(m_UpstreamGetRequestTiming); + + if (GetUpstreamCacheSingleResult UpstreamResult = + m_UpstreamCache.GetCacheRecord(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, AcceptType); + UpstreamResult.Status.Success) + { + Success = true; + + ClientResultValue.Value = UpstreamResult.Value; + ClientResultValue.Value.SetContentType(AcceptType); + + if (AcceptType == ZenContentType::kBinary || AcceptType == ZenContentType::kCbObject) + { + if (AcceptType == ZenContentType::kCbObject) + { + const CbValidateError ValidationResult = ValidateCompactBinary(UpstreamResult.Value, CbValidateMode::All); + if (ValidationResult != CbValidateError::None) + { + Success = false; + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid compact binary object from upstream", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + + // We do not do anything to the returned object for SkipData, only package attachments are cut when skipping data + } + + if (Success && StoreLocal) + { + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = + m_CacheStore + .Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, ClientResultValue, {}, Overwrite, nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + } + } + } + else if (AcceptType == ZenContentType::kCbPackage) + { + CbPackage Package; + if (Package.TryLoad(ClientResultValue.Value)) + { + CbObject CacheRecord = Package.GetObject(); + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector<IoHash> ReferencedAttachments; + std::vector<IoBuffer> WriteAttachmentBuffers; + WriteAttachmentBuffers.reserve(NumAttachments); + std::vector<IoHash> WriteRawHashes; + WriteRawHashes.reserve(NumAttachments); + + CacheRecord.IterateAttachments([this, + &Package, + &Ref, + &WriteAttachmentBuffers, + &WriteRawHashes, + &ReferencedAttachments, + &Count, + QueryLocal, + StoreLocal, + SkipData](CbFieldView HashView) { + IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) + { + if (StoreLocal) + { + const CompressedBuffer& Chunk = Attachment->AsCompressedBinary(); + WriteAttachmentBuffers.push_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Attachment->GetHash()); + } + Count.Valid++; + } + else + { + ZEN_WARN("Uncompressed value '{}' from upstream cache record '{}/{}'", + Hash, + Ref.BucketSegment, + Ref.HashKey); + Count.Invalid++; + } + } + else if (QueryLocal) + { + if (SkipData) + { + if (m_CidStore.ContainsChunk(Hash)) + { + Count.Valid++; + } + } + else if (IoBuffer Chunk = m_CidStore.FindChunkByCid(Hash)) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Chunk)); + if (Compressed) + { + Package.AddAttachment(CbAttachment(Compressed, Hash)); + Count.Valid++; + } + else + { + ZEN_WARN("Uncompressed value '{}' stored in local cache '{}/{}'", Hash, Ref.BucketSegment, Ref.HashKey); + Count.Invalid++; + } + } + } + Count.Total++; + }); + + if ((Count.Valid == Count.Total) || PartialRecord) + { + ZenCacheValue CacheValue; + CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); + CacheValue.Value.SetContentType(ZenContentType::kCbObject); + + if (StoreLocal) + { + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + CacheValue, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status == zen::PutStatus::Success) + { + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) + { + std::vector<CidStore::InsertResult> InsertResults = + m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& Result : InsertResults) + { + if (Result.New) + { + Count.New++; + } + } + } + + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; + } + } + + BinaryWriter MemStream; + if (SkipData) + { + // Save a package containing only the object. + CbPackage(Package.GetObject()).Save(MemStream); + } + else + { + Package.Save(MemStream); + } + + ClientResultValue.Value = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + ClientResultValue.Value.SetContentType(ZenContentType::kCbPackage); + } + else + { + Success = false; + ZEN_WARN("Get - '{}/{}' '{}' FAILED, attachments missing in upstream package", + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + } + else + { + Success = false; + ZEN_WARN("Get - '{}/{}/{}' '{}' FAILED, invalid upstream package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType)); + } + } + } + + if (Success) + { + ZEN_DEBUG("GETCACHERECORD HIT - '{}/{}/{}' {} '{}' (UPSTREAM) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(ClientResultValue.Value.Size()), + ToString(ClientResultValue.Value.GetContentType()), + NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); + + m_CacheStats.HitCount++; + m_CacheStats.UpstreamHitCount++; + + if (SkipData && AcceptType == ZenContentType::kBinary) + { + AsyncRequest.WriteResponse(HttpResponseCode::OK); + } + else + { + // Other methods modify ClientResultValue to a version that has skipped the data but keeps the Object and optionally + // metadata. + AsyncRequest.WriteResponse(HttpResponseCode::OK, ClientResultValue.Value.GetContentType(), ClientResultValue.Value); + } + } + else + { + ZEN_DEBUG("GETCACHERECORD MISS - '{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(AcceptType), + NiceLatencyNs((LocalElapsedTimeUs + Timer.GetElapsedTimeUs()) * 1000)); + m_CacheStats.MissCount++; + AsyncRequest.WriteResponse(HttpResponseCode::NotFound); + } + }); +} + +void +HttpStructuredCacheService::HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + IoBuffer Body = Request.ReadPayload(); + + if (!Body || Body.Size() == 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + if (!AreDiskWritesAllowed()) + { + return Request.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + auto WriteFailureResponse = [&Request](const ZenCacheStore::PutResult& PutResult) { + ZEN_UNUSED(PutResult); + + HttpResponseCode ResponseCode = HttpResponseCode::InternalServerError; + switch (PutResult.Status) + { + case zen::PutStatus::Conflict: + ResponseCode = HttpResponseCode::Conflict; + break; + case zen::PutStatus::Invalid: + ResponseCode = HttpResponseCode::BadRequest; + break; + } + + if (PutResult.Details) + { + Request.WriteResponse(ResponseCode, PutResult.Details); + } + return Request.WriteResponse(ResponseCode); + }; + + const HttpContentType ContentType = Request.RequestContentType(); + + Body.SetContentType(ContentType); + + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + Stopwatch Timer; + + if (ContentType == HttpContentType::kBinary || ContentType == HttpContentType::kCompressedBinary) + { + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = Body.GetSize(); + if (ContentType == HttpContentType::kCompressedBinary) + { + if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Payload is not a valid compressed binary"sv); + } + } + else + { + RawHash = IoHash::HashBuffer(SharedBuffer(Body)); + } + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body, .RawSize = RawSize, .RawHash = RawHash}, + {}, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + if (HasUpstream && EnumHasAllFlags(PolicyFromUrl, CachePolicy::StoreRemote)) + { + m_UpstreamCache.EnqueueUpstream({.Type = ContentType, .Namespace = Ref.Namespace, .Key = {Ref.BucketSegment, Ref.HashKey}}); + } + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + Request.WriteResponse(HttpResponseCode::Created); + } + else if (ContentType == HttpContentType::kCbObject) + { + const CbValidateError ValidationResult = ValidateCompactBinary(MemoryView(Body.GetData(), Body.GetSize()), CbValidateMode::All); + + if (ValidationResult != CbValidateError::None) + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid compact binary", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Compact binary validation failed"sv); + } + + Body.SetContentType(ZenContentType::kCbObject); + + CbObjectView CacheRecord(Body.Data()); + std::vector<IoHash> ValidAttachments; + std::vector<IoHash> ReferencedAttachments; + int32_t TotalCount = 0; + + CacheRecord.IterateAttachments([this, &TotalCount, &ValidAttachments, &ReferencedAttachments](CbFieldView AttachmentHash) { + const IoHash Hash = AttachmentHash.AsHash(); + ReferencedAttachments.push_back(Hash); + if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + } + TotalCount++; + }); + + const bool Overwrite = !EnumHasAllFlags(PolicyFromUrl, CachePolicy::QueryLocal); + + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = m_CacheStore.Put(RequestContext, + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + {.Value = Body}, + ReferencedAttachments, + Overwrite, + nullptr); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}' attachments '{}/{}' (valid/total) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.Size()), + ToString(ContentType), + TotalCount, + ValidAttachments.size(), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const bool IsPartialRecord = TotalCount != static_cast<int32_t>(ValidAttachments.size()); + + CachePolicy Policy = PolicyFromUrl; + if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) + { + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbObject, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); + } + + Request.WriteResponse(HttpResponseCode::Created); + } + else if (ContentType == HttpContentType::kCbPackage) + { + CbPackage Package; + + if (!Package.TryLoad(Body)) + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, invalid package", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(ContentType)); + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"sv); + } + CachePolicy Policy = PolicyFromUrl; + + CbObject CacheRecord = Package.GetObject(); + + AttachmentCount Count; + size_t NumAttachments = Package.GetAttachments().size(); + std::vector<IoHash> ValidAttachments; + std::vector<IoHash> ReferencedAttachments; + ValidAttachments.reserve(NumAttachments); + std::vector<IoBuffer> WriteAttachmentBuffers; + std::vector<IoHash> WriteRawHashes; + WriteAttachmentBuffers.reserve(NumAttachments); + WriteRawHashes.reserve(NumAttachments); + + CacheRecord.IterateAttachments( + [this, &Ref, &Package, &WriteAttachmentBuffers, &WriteRawHashes, &ValidAttachments, &ReferencedAttachments, &Count]( + CbFieldView HashView) { + const IoHash Hash = HashView.AsHash(); + ReferencedAttachments.push_back(Hash); + if (const CbAttachment* Attachment = Package.FindAttachment(Hash)) + { + if (Attachment->IsCompressedBinary()) + { + WriteAttachmentBuffers.emplace_back(Attachment->AsCompressedBinary().GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.push_back(Hash); + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + else + { + ZEN_WARN("PUTCACHERECORD - '{}/{}/{}' '{}' FAILED, attachment '{}' is not compressed", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + ToString(HttpContentType::kCbPackage), + Hash); + Count.Invalid++; + } + } + else if (m_CidStore.ContainsChunk(Hash)) + { + ValidAttachments.emplace_back(Hash); + Count.Valid++; + } + Count.Total++; + }); + + if (Count.Invalid > 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid attachment(s)"sv); + } + + const bool Overwrite = !EnumHasAllFlags(Policy, CachePolicy::QueryLocal); + + ZenCacheValue CacheValue; + CacheValue.Value = CacheRecord.GetBuffer().AsIoBuffer(); + CacheValue.Value.SetContentType(ZenContentType::kCbObject); + // TODO: Propagation for rejected PUTs + ZenCacheStore::PutResult PutResult = + m_CacheStore.Put(RequestContext, Ref.Namespace, Ref.BucketSegment, Ref.HashKey, CacheValue, ReferencedAttachments, Overwrite); + if (PutResult.Status != zen::PutStatus::Success) + { + return WriteFailureResponse(PutResult); + } + m_CacheStats.WriteCount++; + + if (!WriteAttachmentBuffers.empty()) + { + std::vector<CidStore::InsertResult> InsertResults = m_CidStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (const CidStore::InsertResult& InsertResult : InsertResults) + { + if (InsertResult.New) + { + Count.New++; + } + } + WriteAttachmentBuffers = {}; + WriteRawHashes = {}; + } + + ZEN_DEBUG("PUTCACHERECORD - '{}/{}/{}' {} '{}', attachments '{}/{}/{}' (new/valid/total) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + NiceBytes(Body.GetSize()), + ToString(ContentType), + Count.New, + Count.Valid, + Count.Total, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const bool IsPartialRecord = Count.Valid != Count.Total; + + if (HasUpstream && EnumHasAllFlags(Policy, CachePolicy::StoreRemote) && !IsPartialRecord) + { + m_UpstreamCache.EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .Namespace = Ref.Namespace, + .Key = {Ref.BucketSegment, Ref.HashKey}, + .ValueContentIds = std::move(ValidAttachments)}); + } + + Request.WriteResponse(HttpResponseCode::Created); + } + else + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Content-Type invalid"sv); + } +} + +void +HttpStructuredCacheService::HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + switch (Request.RequestVerb()) + { + case HttpVerb::kHead: + case HttpVerb::kGet: + HandleGetCacheChunk(Request, Ref, PolicyFromUrl); + break; + case HttpVerb::kPut: + HandlePutCacheChunk(Request, Ref, PolicyFromUrl); + break; + default: + break; + } +} + +void +HttpStructuredCacheService::HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + Stopwatch Timer; + + IoBuffer Value = m_CidStore.FindChunkByCid(Ref.ValueContentId); + const UpstreamEndpointInfo* Source = nullptr; + CachePolicy Policy = PolicyFromUrl; + + const bool HasUpstream = m_UpstreamCache.IsActive(); + { + const bool QueryUpstream = HasUpstream && !Value && EnumHasAllFlags(Policy, CachePolicy::QueryRemote); + + if (QueryUpstream) + { + if (GetUpstreamCacheSingleResult UpstreamResult = + m_UpstreamCache.GetCacheChunk(Ref.Namespace, {Ref.BucketSegment, Ref.HashKey}, Ref.ValueContentId); + UpstreamResult.Status.Success) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(UpstreamResult.Value, RawHash, RawSize)) + { + if (RawHash == Ref.ValueContentId) + { + if (AreDiskWritesAllowed()) + { + m_CidStore.AddChunk(UpstreamResult.Value, RawHash); + } + Source = UpstreamResult.Source; + } + else + { + ZEN_WARN("got missmatching upstream cache value"); + } + } + else + { + ZEN_WARN("got uncompressed upstream cache value"); + } + } + } + } + + if (!Value) + { + ZEN_DEBUG("GETCACHECHUNK MISS - '{}/{}/{}/{}' '{}' in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + ToString(Request.AcceptContentType()), + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + m_CacheStats.MissCount++; + return Request.WriteResponse(HttpResponseCode::NotFound); + } + + ZEN_DEBUG("GETCACHECHUNK HIT - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + NiceBytes(Value.Size()), + ToString(Value.GetContentType()), + Source ? Source->Url : "LOCAL"sv, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + m_CacheStats.HitCount++; + if (Source) + { + m_CacheStats.UpstreamHitCount++; + } + + if (EnumHasAllFlags(Policy, CachePolicy::SkipData)) + { + Request.WriteResponse(HttpResponseCode::OK); + } + else + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Value); + } +} + +void +HttpStructuredCacheService::HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl) +{ + // Note: Individual cacherecord values are not propagated upstream until a valid cache record has been stored + ZEN_UNUSED(PolicyFromUrl); + + Stopwatch Timer; + + IoBuffer Body = Request.ReadPayload(); + + if (!Body || Body.Size() == 0) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + if (!AreDiskWritesAllowed()) + { + return Request.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + Body.SetContentType(Request.RequestContentType()); + + IoHash RawHash; + uint64_t RawSize; + if (!CompressedBuffer::ValidateCompressedHeader(Body, RawHash, RawSize)) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Attachments must be compressed"sv); + } + + if (RawHash != Ref.ValueContentId) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "ValueContentId does not match attachment hash"sv); + } + + CidStore::InsertResult Result = m_CidStore.AddChunk(Body, RawHash); + + ZEN_DEBUG("PUTCACHECHUNK - '{}/{}/{}/{}' {} '{}' ({}) in {}", + Ref.Namespace, + Ref.BucketSegment, + Ref.HashKey, + Ref.ValueContentId, + NiceBytes(Body.Size()), + ToString(Body.GetContentType()), + Result.New ? "NEW" : "OLD", + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + + const HttpResponseCode ResponseCode = Result.New ? HttpResponseCode::Created : HttpResponseCode::OK; + + Request.WriteResponse(ResponseCode); +} + +void +HttpStructuredCacheService::ReplayRequestRecorder(const CacheRequestContext& Context, + cache::IRpcRequestReplayer& Replayer, + uint32_t ThreadCount) +{ + WorkerThreadPool WorkerPool(ThreadCount); + uint64_t RequestCount = Replayer.GetRequestCount(); + Stopwatch Timer; + auto _ = MakeGuard([&]() { ZEN_INFO("Replayed {} requests in {}", RequestCount, NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); }); + std::atomic<bool> AbortFlag; + std::atomic<bool> PauseFlag; + ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + ZEN_INFO("Replaying {} requests", RequestCount); + for (uint64_t RequestIndex = 0; RequestIndex < RequestCount; ++RequestIndex) + { + if (AbortFlag) + { + break; + } + Work.ScheduleWork(WorkerPool, [this, &Context, &Replayer, RequestIndex](std::atomic<bool>& AbortFlag) { + IoBuffer Body; + zen::cache::RecordedRequestInfo RequestInfo = Replayer.GetRequest(RequestIndex, /* out */ Body); + + if (AbortFlag) + { + return; + } + + if (Body) + { + uint32_t AcceptMagic = 0; + RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; + int TargetPid = 0; + CbPackage RpcResult; + if (m_RpcHandler.HandleRpcRequest(Context, + /* UriNamespace */ {}, + RequestInfo.ContentType, + std::move(Body), + AcceptMagic, + AcceptFlags, + TargetPid, + RpcResult) == CacheRpcHandler::RpcResponseCode::OK) + { + if (AcceptMagic == kCbPkgMagic) + { + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(Context.SessionId, TargetPid); + } + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); + ZEN_ASSERT(RpcResponseBuffer.GetSize() > 0); + } + else + { + BinaryWriter MemStream; + RpcResult.Save(MemStream); + IoBuffer RpcResponseBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize()); + ZEN_ASSERT(RpcResponseBuffer.Size() > 0); + } + } + } + }); + } + Work.Wait(10000, [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { + ZEN_UNUSED(IsAborted, IsPaused); + ZEN_INFO("Replayed {} of {} requests, elapsed {}", + RequestCount - PendingWork, + RequestCount, + NiceLatencyNs(Timer.GetElapsedTimeUs() * 1000)); + }); +} + +void +HttpStructuredCacheService::HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace) +{ + ZEN_MEMSCOPE(GetCacheRpcTag()); + + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest"); + + const bool HasUpstream = m_UpstreamCache.IsActive(); + + switch (Request.RequestVerb()) + { + case HttpVerb::kPost: + { + CacheRequestContext RequestContext = {.SessionId = Request.SessionId(), .RequestId = Request.RequestId()}; + + const HttpContentType ContentType = Request.RequestContentType(); + const HttpContentType AcceptType = Request.AcceptContentType(); + + if ((ContentType != HttpContentType::kCbObject && ContentType != HttpContentType::kCbPackage) || + AcceptType != HttpContentType::kCbPackage) + { + m_CacheStats.BadRequestCount++; + return Request.WriteResponse(HttpResponseCode::BadRequest); + } + + auto HandleRpc = [this, + RequestContext, + Body = Request.ReadPayload(), + ContentType, + AcceptType, + UriNamespaceString = std::string{UriNamespace}](HttpServerRequest& AsyncRequest) mutable { + if (m_RequestRecordingEnabled) + { + RwLock::SharedLockScope _(m_RequestRecordingLock); + if (m_RequestRecorder) + { + m_RequestRecorder->RecordRequest( + {.ContentType = ContentType, .AcceptType = AcceptType, .SessionId = RequestContext.SessionId}, + Body); + } + } + + uint32_t AcceptMagic = 0; + RpcAcceptOptions AcceptFlags = RpcAcceptOptions::kNone; + int TargetProcessId = 0; + CbPackage RpcResult; + + CacheRpcHandler::RpcResponseCode ResultCode = m_RpcHandler.HandleRpcRequest(RequestContext, + UriNamespaceString, + ContentType, + std::move(Body), + /* out */ AcceptMagic, + /* out */ AcceptFlags, + /* out */ TargetProcessId, + /* out */ RpcResult); + + HttpResponseCode HttpResultCode = HttpResponseCode(int(ResultCode)); + + if (!IsHttpSuccessCode(HttpResultCode)) + { + return AsyncRequest.WriteResponse(HttpResultCode); + } + + if (AcceptMagic == kCbPkgMagic) + { + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(RequestContext.SessionId, TargetProcessId); + } + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(RpcResult, Flags, TargetProcessHandle); + AsyncRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + else + { + BinaryWriter MemStream; + RpcResult.Save(MemStream); + + AsyncRequest.WriteResponse(HttpResponseCode::OK, + HttpContentType::kCbPackage, + IoBuffer(IoBuffer::Wrap, MemStream.GetData(), MemStream.GetSize())); + } + }; + + if (HasUpstream) + { + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponseAsync"); + Request.WriteResponseAsync(std::move(HandleRpc)); + } + else + { + ZEN_TRACE_CPU("z$::Http::HandleRpcRequest::WriteResponse"); + HandleRpc(Request); + } + } + break; + + default: + m_CacheStats.BadRequestCount++; + Request.WriteResponse(HttpResponseCode::BadRequest); + break; + } +} + +void +HttpStructuredCacheService::HandleStatsRequest(HttpServerRequest& Request) +{ + ZEN_MEMSCOPE(GetCacheHttpTag()); + + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + const uint64_t HitCount = m_CacheStats.HitCount; + const uint64_t UpstreamHitCount = m_CacheStats.UpstreamHitCount; + const uint64_t MissCount = m_CacheStats.MissCount; + const uint64_t WriteCount = m_CacheStats.WriteCount; + const uint64_t BadRequestCount = m_CacheStats.BadRequestCount; + struct CidStoreStats StoreStats = m_CidStore.Stats(); + const uint64_t ChunkHitCount = StoreStats.HitCount; + const uint64_t ChunkMissCount = StoreStats.MissCount; + const uint64_t ChunkWriteCount = StoreStats.WriteCount; + const uint64_t TotalCount = HitCount + MissCount; + + const uint64_t RpcRequests = m_CacheStats.RpcRequests; + const uint64_t RpcRecordRequests = m_CacheStats.RpcRecordRequests; + const uint64_t RpcRecordBatchRequests = m_CacheStats.RpcRecordBatchRequests; + const uint64_t RpcValueRequests = m_CacheStats.RpcValueRequests; + const uint64_t RpcValueBatchRequests = m_CacheStats.RpcValueBatchRequests; + const uint64_t RpcChunkRequests = m_CacheStats.RpcChunkRequests; + const uint64_t RpcChunkBatchRequests = m_CacheStats.RpcChunkBatchRequests; + + const CidStoreSize CidSize = m_CidStore.TotalSize(); + const CacheStoreSize CacheSize = m_CacheStore.TotalSize(); + + bool ShowCidStoreStats = Request.GetQueryParams().GetValue("cidstorestats") == "true"; + bool ShowCacheStoreStats = Request.GetQueryParams().GetValue("cachestorestats") == "true"; + + CidStoreStats CidStoreStats = {}; + if (ShowCidStoreStats) + { + CidStoreStats = m_CidStore.Stats(); + } + ZenCacheStore::CacheStoreStats CacheStoreStats = {}; + if (ShowCacheStoreStats) + { + CacheStoreStats = m_CacheStore.Stats(); + } + + Cbo.BeginObject("cache"); + { + Cbo << "badrequestcount" << BadRequestCount; + Cbo.BeginObject("rpc"); + Cbo << "count" << RpcRequests; + Cbo << "ops" << RpcRecordBatchRequests + RpcValueBatchRequests + RpcChunkBatchRequests; + Cbo.BeginObject("records"); + Cbo << "count" << RpcRecordRequests; + Cbo << "ops" << RpcRecordBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("values"); + Cbo << "count" << RpcValueRequests; + Cbo << "ops" << RpcValueBatchRequests; + Cbo.EndObject(); + Cbo.BeginObject("chunks"); + Cbo << "count" << RpcChunkRequests; + Cbo << "ops" << RpcChunkBatchRequests; + Cbo.EndObject(); + Cbo.EndObject(); + + Cbo.BeginObject("size"); + { + Cbo << "disk" << CacheSize.DiskSize; + Cbo << "memory" << CacheSize.MemorySize; + } + Cbo.EndObject(); + + Cbo << "hits" << HitCount << "misses" << MissCount << "writes" << WriteCount; + Cbo << "hit_ratio" << (TotalCount > 0 ? (double(HitCount) / double(TotalCount)) : 0.0); + + if (m_UpstreamCache.IsActive()) + { + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + Cbo << "upstream_hits" << m_CacheStats.UpstreamHitCount; + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + Cbo << "upstream_ratio" << (HitCount > 0 ? (double(UpstreamHitCount) / double(HitCount)) : 0.0); + } + + Cbo << "cidhits" << ChunkHitCount << "cidmisses" << ChunkMissCount << "cidwrites" << ChunkWriteCount; + + if (ShowCacheStoreStats) + { + Cbo.BeginObject("store"); + Cbo << "hits" << CacheStoreStats.HitCount << "misses" << CacheStoreStats.MissCount << "writes" << CacheStoreStats.WriteCount + << "rejected_writes" << CacheStoreStats.RejectedWriteCount << "rejected_reads" << CacheStoreStats.RejectedReadCount; + const uint64_t StoreTotal = CacheStoreStats.HitCount + CacheStoreStats.MissCount; + Cbo << "hit_ratio" << (StoreTotal > 0 ? (double(CacheStoreStats.HitCount) / double(StoreTotal)) : 0.0); + EmitSnapshot("read", CacheStoreStats.GetOps, Cbo); + EmitSnapshot("write", CacheStoreStats.PutOps, Cbo); + if (!CacheStoreStats.NamespaceStats.empty()) + { + Cbo.BeginArray("namespaces"); + for (const ZenCacheStore::NamedNamespaceStats& NamespaceStats : CacheStoreStats.NamespaceStats) + { + Cbo.BeginObject(); + Cbo.AddString("namespace", NamespaceStats.NamespaceName); + Cbo << "hits" << NamespaceStats.Stats.HitCount << "misses" << NamespaceStats.Stats.MissCount << "writes" + << NamespaceStats.Stats.WriteCount; + const uint64_t NamespaceTotal = NamespaceStats.Stats.HitCount + NamespaceStats.Stats.MissCount; + Cbo << "hit_ratio" << (NamespaceTotal > 0 ? (double(NamespaceStats.Stats.HitCount) / double(NamespaceTotal)) : 0.0); + EmitSnapshot("read", NamespaceStats.Stats.GetOps, Cbo); + EmitSnapshot("write", NamespaceStats.Stats.PutOps, Cbo); + Cbo.BeginObject("size"); + { + Cbo << "disk" << NamespaceStats.Stats.DiskStats.DiskSize; + Cbo << "memory" << NamespaceStats.Stats.DiskStats.MemorySize; + } + Cbo.EndObject(); + if (!NamespaceStats.Stats.DiskStats.BucketStats.empty()) + { + Cbo.BeginArray("buckets"); + for (const ZenCacheDiskLayer::NamedBucketStats& BucketStats : NamespaceStats.Stats.DiskStats.BucketStats) + { + Cbo.BeginObject(); + Cbo.AddString("bucket", BucketStats.BucketName); + if (BucketStats.Stats.DiskSize != 0 || BucketStats.Stats.MemorySize != 0) + { + Cbo.BeginObject("size"); + { + Cbo << "disk" << BucketStats.Stats.DiskSize; + Cbo << "memory" << BucketStats.Stats.MemorySize; + } + Cbo.EndObject(); + } + + if (BucketStats.Stats.DiskSize == 0 && BucketStats.Stats.DiskHitCount == 0 && + BucketStats.Stats.DiskMissCount == 0 && BucketStats.Stats.DiskWriteCount == 0 && + BucketStats.Stats.MemoryHitCount == 0 && BucketStats.Stats.MemoryMissCount == 0 && + BucketStats.Stats.MemoryWriteCount == 0) + { + Cbo.EndObject(); + continue; + } + + const uint64_t BucketDiskTotal = BucketStats.Stats.DiskHitCount + BucketStats.Stats.DiskMissCount; + if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0) + { + Cbo << "hits" << BucketStats.Stats.DiskHitCount << "misses" << BucketStats.Stats.DiskMissCount << "writes" + << BucketStats.Stats.DiskWriteCount; + Cbo << "hit_ratio" + << (BucketDiskTotal > 0 ? (double(BucketStats.Stats.DiskHitCount) / double(BucketDiskTotal)) : 0.0); + } + + const uint64_t BucketMemoryTotal = BucketStats.Stats.MemoryHitCount + BucketStats.Stats.MemoryMissCount; + if (BucketMemoryTotal != 0 || BucketStats.Stats.MemoryWriteCount != 0) + { + Cbo << "mem_hits" << BucketStats.Stats.MemoryHitCount << "mem_misses" << BucketStats.Stats.MemoryMissCount + << "mem_writes" << BucketStats.Stats.MemoryWriteCount; + Cbo << "mem_hit_ratio" + << (BucketMemoryTotal > 0 ? (double(BucketStats.Stats.MemoryHitCount) / double(BucketMemoryTotal)) + : 0.0); + } + + if (BucketDiskTotal != 0 || BucketStats.Stats.DiskWriteCount != 0 || BucketMemoryTotal != 0 || + BucketStats.Stats.MemoryWriteCount != 0) + { + EmitSnapshot("read", BucketStats.Stats.GetOps, Cbo); + EmitSnapshot("write", BucketStats.Stats.PutOps, Cbo); + } + + Cbo.EndObject(); + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + } + Cbo.EndObject(); + } + Cbo.EndObject(); + } + + if (m_UpstreamCache.IsActive()) + { + EmitSnapshot("upstream_gets", m_UpstreamGetRequestTiming, Cbo); + Cbo.BeginObject("upstream"); + { + m_UpstreamCache.GetStatus(Cbo); + } + Cbo.EndObject(); + } + + Cbo.BeginObject("cid"); + { + Cbo.BeginObject("size"); + { + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; + } + Cbo.EndObject(); + + if (ShowCidStoreStats) + { + Cbo.BeginObject("store"); + Cbo << "hits" << CidStoreStats.HitCount << "misses" << CidStoreStats.MissCount << "writes" << CidStoreStats.WriteCount; + EmitSnapshot("read", CidStoreStats.FindChunkOps, Cbo); + EmitSnapshot("write", CidStoreStats.AddChunkOps, Cbo); + // EmitSnapshot("exists", CidStoreStats.ContainChunkOps, Cbo); + Cbo.EndObject(); + } + } + Cbo.EndObject(); + + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpStructuredCacheService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +bool +HttpStructuredCacheService::AreDiskWritesAllowed() const +{ + return (m_DiskWriteBlocker == nullptr || m_DiskWriteBlocker->AreDiskWritesAllowed()); +} + +} // namespace zen diff --git a/src/zenserver/storage/cache/httpstructuredcache.h b/src/zenserver/storage/cache/httpstructuredcache.h new file mode 100644 index 000000000..a157148c9 --- /dev/null +++ b/src/zenserver/storage/cache/httpstructuredcache.h @@ -0,0 +1,138 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> +#include <zenstore/cache/cache.h> +#include <zenstore/cache/cacherpc.h> +#include <zenutil/openprocesscache.h> + +#include <memory> +#include <vector> + +namespace zen { + +struct CacheChunkRequest; +struct CacheKeyRequest; +struct PutRequestData; + +class CidStore; +class CbObjectView; +class DiskWriteBlocker; +class HttpStructuredCacheService; +class ScrubContext; +class UpstreamCache; +class ZenCacheStore; + +enum class CachePolicy : uint32_t; +enum class RpcAcceptOptions : uint16_t; + +namespace cache { + class IRpcRequestReplayer; + class IRpcRequestRecorder; + namespace detail { + struct RecordBody; + struct ChunkRequest; + } // namespace detail +} // namespace cache + +/** + * Structured cache service. Imposes constraints on keys, supports blobs and + * structured values + * + * Keys are structured as: + * + * {BucketId}/{KeyHash} + * + * Where BucketId is a lower-case alphanumeric string, and KeyHash is a 40-character + * hexadecimal sequence. The hash value may be derived in any number of ways, it's + * up to the application to pick an approach. + * + * Values may be structured or unstructured. Structured values are encoded using Unreal + * Engine's compact binary encoding (see CbObject) + * + * Additionally, attachments may be addressed as: + * + * {BucketId}/{KeyHash}/{ValueHash} + * + * Where the two initial components are the same as for the main endpoint + * + * The storage strategy is as follows: + * + * - Structured values are stored in a dedicated backing store per bucket + * - Unstructured values and attachments are stored in the CAS pool + * + */ + +class HttpStructuredCacheService : public HttpService, public IHttpStatsProvider, public IHttpStatusProvider +{ +public: + HttpStructuredCacheService(ZenCacheStore& InCacheStore, + CidStore& InCidStore, + HttpStatsService& StatsService, + HttpStatusService& StatusService, + UpstreamCache& UpstreamCache, + const DiskWriteBlocker* InDiskWriteBlocker, + OpenProcessCache& InOpenProcessCache); + ~HttpStructuredCacheService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + void Flush(); + +private: + struct CacheRef + { + std::string Namespace; + std::string BucketSegment; + IoHash HashKey; + IoHash ValueContentId; + }; + + void HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheRecord(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleCacheChunkRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleGetCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandlePutCacheChunk(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy PolicyFromUrl); + void HandleRpcRequest(HttpServerRequest& Request, std::string_view UriNamespace); + void HandleDetailsRequest(HttpServerRequest& Request); + + void HandleCacheRequest(HttpServerRequest& Request); + void HandleCacheNamespaceRequest(HttpServerRequest& Request, std::string_view Namespace); + void HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Namespace, std::string_view Bucket); + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + + bool AreDiskWritesAllowed() const; + + LoggerRef Log() { return m_Log; } + LoggerRef m_Log; + ZenCacheStore& m_CacheStore; + HttpStatsService& m_StatsService; + HttpStatusService& m_StatusService; + CidStore& m_CidStore; + UpstreamCache& m_UpstreamCache; + metrics::OperationTiming m_HttpRequests; + metrics::OperationTiming m_UpstreamGetRequestTiming; + CacheStats m_CacheStats; + const DiskWriteBlocker* m_DiskWriteBlocker = nullptr; + OpenProcessCache& m_OpenProcessCache; + CacheRpcHandler m_RpcHandler; + + void ReplayRequestRecorder(const CacheRequestContext& Context, cache::IRpcRequestReplayer& Replayer, uint32_t ThreadCount); + + // This exists to avoid taking locks when recording is not enabled + std::atomic_bool m_RequestRecordingEnabled{false}; + + // This lock should be taken in SHARED mode when calling into the recorder, + // and taken in EXCLUSIVE mode whenever the recorder is created or destroyed + RwLock m_RequestRecordingLock; + std::unique_ptr<cache::IRpcRequestRecorder> m_RequestRecorder; +}; + +} // namespace zen diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp new file mode 100644 index 000000000..3f40bf616 --- /dev/null +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -0,0 +1,618 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "objectstore.h" + +#include <zencore/base64.h> +#include <zencore/basicfile.h> +#include <zencore/compactbinaryvalue.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/string.h> +#include <zencore/trace.h> +#include "zencore/compactbinary.h" +#include "zencore/compactbinarybuilder.h" +#include "zenhttp/httpcommon.h" +#include "zenhttp/httpserver.h" + +#include <filesystem> +#include <thread> + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "obj"sv); + +class CbXmlWriter +{ +public: + explicit CbXmlWriter(StringBuilderBase& InBuilder) : Builder(InBuilder) + { + Builder.Append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); + Builder << LINE_TERMINATOR_ANSI; + } + + void WriteField(CbFieldView Field) + { + using namespace std::literals; + + bool SkipEndTag = false; + const std::u8string_view Tag = Field.GetU8Name(); + + AppendBeginTag(Tag); + + switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + { + case CbFieldType::Null: + Builder << "Null"sv; + break; + case CbFieldType::Object: + case CbFieldType::UniformObject: + { + for (CbFieldView It : Field) + { + WriteField(It); + } + } + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + { + bool FirstField = true; + for (CbFieldView It : Field) + { + if (!FirstField) + AppendBeginTag(Tag); + + WriteField(It); + AppendEndTag(Tag); + FirstField = false; + } + SkipEndTag = true; + } + break; + case CbFieldType::Binary: + AppendBase64String(Accessor.AsBinary()); + break; + case CbFieldType::String: + Builder << Accessor.AsU8String(); + break; + case CbFieldType::IntegerPositive: + Builder << Accessor.AsIntegerPositive(); + break; + case CbFieldType::IntegerNegative: + Builder << Accessor.AsIntegerNegative(); + break; + case CbFieldType::Float32: + { + const float Value = Accessor.AsFloat32(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.9g}", Value)); + } + else + { + Builder << "Null"sv; + } + } + break; + case CbFieldType::Float64: + { + const double Value = Accessor.AsFloat64(); + if (std::isfinite(Value)) + { + Builder.Append(fmt::format("{:.17g}", Value)); + } + else + { + Builder << "null"sv; + } + } + break; + case CbFieldType::BoolFalse: + Builder << "False"sv; + break; + case CbFieldType::BoolTrue: + Builder << "True"sv; + break; + case CbFieldType::ObjectAttachment: + case CbFieldType::BinaryAttachment: + { + Accessor.AsAttachment().ToHexString(Builder); + } + break; + case CbFieldType::Hash: + { + Accessor.AsHash().ToHexString(Builder); + } + break; + case CbFieldType::Uuid: + { + Accessor.AsUuid().ToString(Builder); + } + break; + case CbFieldType::DateTime: + Builder << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + break; + case CbFieldType::TimeSpan: + { + const TimeSpan Span(Accessor.AsTimeSpanTicks()); + if (Span.GetDays() == 0) + { + Builder << Span.ToString("%h:%m:%s.%n"); + } + else + { + Builder << Span.ToString("%d.%h:%m:%s.%n"); + } + break; + } + case CbFieldType::ObjectId: + Accessor.AsObjectId().ToString(Builder); + break; + case CbFieldType::CustomById: + { + CbCustomById Custom = Accessor.AsCustomById(); + + AppendBeginTag(u8"Id"sv); + Builder << Custom.Id; + AppendEndTag(u8"Id"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + case CbFieldType::CustomByName: + { + CbCustomByName Custom = Accessor.AsCustomByName(); + + AppendBeginTag(u8"Name"sv); + Builder << Custom.Name; + AppendEndTag(u8"Name"sv); + + AppendBeginTag(u8"Data"sv); + AppendBase64String(Custom.Data); + AppendEndTag(u8"Data"sv); + break; + } + default: + ZEN_ASSERT(false); + break; + } + + if (!SkipEndTag) + AppendEndTag(Tag); + } + +private: + void AppendBeginTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << '<' << Tag << '>'; + } + } + + void AppendEndTag(std::u8string_view Tag) + { + if (!Tag.empty()) + { + Builder << "</"sv << Tag << '>'; + } + } + + void AppendBase64String(MemoryView Value) + { + Builder << '"'; + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast<const uint8_t*>(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + } + +private: + StringBuilderBase& Builder; +}; + +HttpObjectStoreService::HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg) +: m_StatusService(StatusService) +, m_Cfg(std::move(Cfg)) +{ + Inititalize(); + m_StatusService.RegisterHandler("obj", *this); +} + +HttpObjectStoreService::~HttpObjectStoreService() +{ + m_StatusService.UnregisterHandler("obj", *this); +} + +const char* +HttpObjectStoreService::BaseUri() const +{ + return "/obj/"; +} + +void +HttpObjectStoreService::HandleRequest(zen::HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpObjectStoreService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpObjectStoreService::Inititalize() +{ + ZEN_TRACE_CPU("HttpObjectStoreService::Inititalize"); + + namespace fs = std::filesystem; + ZEN_LOG_INFO(LogObj, "Initialzing Object Store in '{}'", m_Cfg.RootDirectory); + + const fs::path BucketsPath = m_Cfg.RootDirectory / "buckets"; + if (!IsDir(BucketsPath)) + { + CreateDirectories(BucketsPath); + } + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { CreateBucket(Request); }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "bucket", + [this](zen::HttpRouterRequest& Request) { DeleteBucket(Request); }, + HttpVerb::kDelete); + + m_Router.RegisterRoute( + "bucket/{path}", + [this](zen::HttpRouterRequest& Request) { + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + + if (IsObject) + { + GetObject(Request, Path); + } + else + { + ListBucket(Request, Path); + } + }, + HttpVerb::kHead | HttpVerb::kGet); + + m_Router.RegisterRoute( + "bucket/{bucket}/{path}", + [this](zen::HttpRouterRequest& Request) { PutObject(Request); }, + HttpVerb::kPost | HttpVerb::kPut); +} + +std::filesystem::path +HttpObjectStoreService::GetBucketDirectory(std::string_view BucketName) +{ + { + std::lock_guard _(BucketsMutex); + + if (const auto It = std::find_if(std::begin(m_Cfg.Buckets), + std::end(m_Cfg.Buckets), + [&BucketName](const auto& Bucket) -> bool { return Bucket.Name == BucketName; }); + It != std::end(m_Cfg.Buckets)) + { + return It->Directory.make_preferred(); + } + } + + return (m_Cfg.RootDirectory / "buckets" / BucketName).make_preferred(); +} + +void +HttpObjectStoreService::CreateBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + if (!IsDir(BucketPath)) + { + CreateDirectories(BucketPath); + ZEN_LOG_INFO(LogObj, "CREATE - new bucket '{}' OK", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Created); + } + } + + ZEN_LOG_INFO(LogObj, "CREATE - existing bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + if (BucketPrefix.empty()) + { + const auto QueryParms = Request.ServerRequest().GetQueryParams(); + if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) + { + BucketPrefix = PrefixParam; + } + } + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); + BucketPrefix.erase(0, BucketPrefix.find_first_not_of('\\')); + + const fs::path BucketRoot = GetBucketDirectory(BucketName); + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + const fs::path FullPath = BucketRoot / RelativeBucketPath; + + struct Visitor : FileSystemTraversal::TreeVisitor + { + Visitor(const std::string_view BucketName, const fs::path& Path, const fs::path& Prefix) : BucketPath(Path) + { + Writer.BeginObject("ListBucketResult"sv); + Writer << "Name"sv << BucketName; + std::string Tmp = Prefix.string(); + std::replace(Tmp.begin(), Tmp.end(), '\\', '/'); + Writer << "Prefix"sv << Tmp; + Writer.BeginArray("Contents"sv); + } + + void VisitFile(const fs::path& Parent, const path_view& File, uint64_t FileSize, uint32_t, uint64_t) override + { + const fs::path FullPath = Parent / fs::path(File); + fs::path RelativePath = fs::relative(FullPath, BucketPath); + + std::string Key = RelativePath.string(); + std::replace(Key.begin(), Key.end(), '\\', '/'); + + Writer.BeginObject(); + Writer << "Key"sv << Key; + Writer << "Size"sv << FileSize; + Writer.EndObject(); + } + + bool VisitDirectory(const std::filesystem::path&, const path_view&, uint32_t) override { return false; } + + CbObject GetResult() + { + Writer.EndArray(); + Writer.EndObject(); + return Writer.Save(); + } + + CbObjectWriter Writer; + fs::path BucketPath; + }; + + Visitor FileVisitor(BucketName, BucketRoot, RelativeBucketPath); + FileSystemTraversal Traversal; + + if (IsDir(FullPath)) + { + std::lock_guard _(BucketsMutex); + Traversal.TraverseFileSystem(FullPath, FileVisitor); + } + CbObject Result = FileVisitor.GetResult(); + + if (Request.ServerRequest().AcceptContentType() == HttpContentType::kJSON) + { + ExtendableStringBuilder<1024> Sb; + return Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, Result.ToJson(Sb).ToView()); + } + + ExtendableStringBuilder<1024> Xml; + CbXmlWriter XmlWriter(Xml); + XmlWriter.WriteField(Result.AsFieldView()); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kXML, Xml.ToView()); +} + +void +HttpObjectStoreService::DeleteBucket(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const CbObject Params = Request.ServerRequest().ReadPayloadObject(); + const std::string_view BucketName = Params["bucketname"].AsString(); + + if (BucketName.empty()) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const fs::path BucketPath = m_Cfg.RootDirectory / "buckets" / BucketName; + { + std::lock_guard _(BucketsMutex); + DeleteDirectories(BucketPath); + } + + ZEN_LOG_INFO(LogObj, "DELETE - bucket '{}' OK", BucketName); + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +void +HttpObjectStoreService::GetObject(zen::HttpRouterRequest& Request, const std::string_view Path) +{ + namespace fs = std::filesystem; + + const auto Sep = Path.find_first_of('/'); + const std::string BucketName{Sep == std::string::npos ? Path : Path.substr(0, Sep)}; + const std::string BucketPrefix{Sep == std::string::npos || Sep == Path.size() - 1 ? std::string() : Path.substr(BucketName.size() + 1)}; + + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "GET - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(BucketPrefix).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "GET - from bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + if (!IsFile(FilePath)) + { + ZEN_LOG_DEBUG(LogObj, "GET - '{}/{}' [FAILED], doesn't exist", BucketName, FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + zen::HttpRanges Ranges; + if (Request.ServerRequest().TryGetRanges(Ranges); Ranges.size() > 1) + { + // Only a single range is supported + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + FileContents File; + { + std::lock_guard _(BucketsMutex); + File = ReadFile(FilePath); + } + + if (File.ErrorCode) + { + ZEN_LOG_WARN(LogObj, + "GET - '{}/{}' [FAILED] ('{}': {})", + BucketName, + FilePath, + File.ErrorCode.category().name(), + File.ErrorCode.value()); + + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + const IoBuffer& FileBuf = File.Data[0]; + + if (Ranges.empty()) + { + const uint64_t TotalServed = TotalBytesServed.fetch_add(FileBuf.Size()) + FileBuf.Size(); + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' ({}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, FileBuf); + } + else + { + const auto Range = Ranges[0]; + const uint64_t RangeSize = 1 + (Range.End - Range.Start); + const uint64_t TotalServed = TotalBytesServed.fetch_add(RangeSize) + RangeSize; + + ZEN_LOG_DEBUG(LogObj, + "GET - '{}/{}' (Range: {}-{}) ({}/{}) [OK] (Served: {})", + BucketName, + RelativeBucketPath, + Range.Start, + Range.End, + NiceBytes(RangeSize), + NiceBytes(FileBuf.Size()), + NiceBytes(TotalServed)); + + MemoryView RangeView = FileBuf.GetView().Mid(Range.Start, RangeSize); + if (RangeView.GetSize() != RangeSize) + { + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + IoBuffer RangeBuf = IoBuffer(IoBuffer::Wrap, RangeView.GetData(), RangeView.GetSize()); + Request.ServerRequest().WriteResponse(HttpResponseCode::PartialContent, HttpContentType::kBinary, RangeBuf); + } +} + +void +HttpObjectStoreService::PutObject(zen::HttpRouterRequest& Request) +{ + namespace fs = std::filesystem; + + const std::string_view BucketName = Request.GetCapture(1); + const fs::path BucketDir = GetBucketDirectory(BucketName); + + if (BucketDir.empty()) + { + ZEN_LOG_DEBUG(LogObj, "PUT - [FAILED], unknown bucket '{}'", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::NotFound); + } + + const fs::path RelativeBucketPath = fs::path(Request.GetCapture(2)).make_preferred(); + + if (RelativeBucketPath.is_absolute() || RelativeBucketPath.string().starts_with("..")) + { + ZEN_LOG_DEBUG(LogObj, "PUT - bucket '{}' [FAILED], invalid file path", BucketName); + return Request.ServerRequest().WriteResponse(HttpResponseCode::Forbidden); + } + + const fs::path FilePath = BucketDir / RelativeBucketPath; + const fs::path FileDirectory = FilePath.parent_path(); + + { + std::lock_guard _(BucketsMutex); + + if (!IsDir(FileDirectory)) + { + CreateDirectories(FileDirectory); + } + + const IoBuffer FileBuf = Request.ServerRequest().ReadPayload(); + + if (FileBuf.Size() == 0) + { + ZEN_LOG_DEBUG(LogObj, "PUT - '{}' [FAILED], empty file", FilePath); + return Request.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); + } + + TemporaryFile::SafeWriteFile(FilePath, FileBuf.GetView()); + + ZEN_LOG_DEBUG(LogObj, + "PUT - '{}' [OK] ({})", + (fs::path(BucketName) / RelativeBucketPath).make_preferred(), + NiceBytes(FileBuf.Size())); + } + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK); +} + +} // namespace zen diff --git a/src/zenserver/storage/objectstore/objectstore.h b/src/zenserver/storage/objectstore/objectstore.h new file mode 100644 index 000000000..44e50e208 --- /dev/null +++ b/src/zenserver/storage/objectstore/objectstore.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> +#include <atomic> +#include <filesystem> +#include <mutex> + +namespace zen { + +class HttpRouterRequest; + +struct ObjectStoreConfig +{ + struct BucketConfig + { + std::string Name; + std::filesystem::path Directory; + }; + + std::filesystem::path RootDirectory; + std::vector<BucketConfig> Buckets; +}; + +class HttpObjectStoreService final : public zen::HttpService, public IHttpStatusProvider +{ +public: + HttpObjectStoreService(HttpStatusService& StatusService, ObjectStoreConfig Cfg); + virtual ~HttpObjectStoreService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + void Inititalize(); + std::filesystem::path GetBucketDirectory(std::string_view BucketName); + void CreateBucket(zen::HttpRouterRequest& Request); + void ListBucket(zen::HttpRouterRequest& Request, const std::string_view Path); + void DeleteBucket(zen::HttpRouterRequest& Request); + void GetObject(zen::HttpRouterRequest& Request, const std::string_view Path); + void PutObject(zen::HttpRouterRequest& Request); + + HttpStatusService& m_StatusService; + ObjectStoreConfig m_Cfg; + std::mutex BucketsMutex; + HttpRequestRouter m_Router; + std::atomic_uint64_t TotalBytesServed{0}; +}; + +} // namespace zen diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp new file mode 100644 index 000000000..1c6b5d6b0 --- /dev/null +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -0,0 +1,3307 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpprojectstore.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryutil.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/memory/llm.h> +#include <zencore/scopeguard.h> +#include <zencore/stream.h> +#include <zencore/trace.h> +#include <zenhttp/packageformat.h> +#include <zenremotestore/projectstore/buildsremoteprojectstore.h> +#include <zenremotestore/projectstore/fileremoteprojectstore.h> +#include <zenremotestore/projectstore/jupiterremoteprojectstore.h> +#include <zenremotestore/projectstore/remoteprojectstore.h> +#include <zenremotestore/projectstore/zenremoteprojectstore.h> +#include <zenstore/oplogreferencedset.h> +#include <zenstore/projectstore.h> +#include <zenstore/zenstore.h> +#include <zenutil/openprocesscache.h> +#include <zenutil/workerpools.h> + +namespace zen { + +const FLLMTag& +GetProjectHttpTag() +{ + static FLLMTag _("http", FLLMTag("project")); + + return _; +} + +void +CSVHeader(bool Details, bool AttachmentDetails, StringBuilderBase& CSVWriter) +{ + if (AttachmentDetails) + { + CSVWriter << "Project, Oplog, LSN, Key, Cid, Size"; + } + else if (Details) + { + CSVWriter << "Project, Oplog, LSN, Key, Size, AttachmentCount, AttachmentsSize"; + } + else + { + CSVWriter << "Project, Oplog, Key"; + } +} + +void +CSVWriteOp(CidStore& CidStore, + std::string_view ProjectId, + std::string_view OplogId, + bool Details, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + StringBuilderBase& CSVWriter) +{ + StringBuilder<32> KeyStringBuilder; + Key.ToString(KeyStringBuilder); + const std::string_view KeyString = KeyStringBuilder.ToView(); + + if (AttachmentDetails) + { + Op.IterateAttachments([&CidStore, &CSVWriter, &ProjectId, &OplogId, LSN, &KeyString](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + CSVWriter << "\r\n" + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << AttachmentHash.ToHexString() + << ", " << gsl::narrow<uint64_t>(Attachment.GetSize()); + }); + } + else if (Details) + { + uint64_t AttachmentCount = 0; + size_t AttachmentsSize = 0; + Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + AttachmentCount++; + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + AttachmentsSize += Attachment.GetSize(); + }); + CSVWriter << "\r\n" + << ProjectId << ", " << OplogId << ", " << LSN.Number << ", " << KeyString << ", " << gsl::narrow<uint64_t>(Op.GetSize()) + << ", " << AttachmentCount << ", " << gsl::narrow<uint64_t>(AttachmentsSize); + } + else + { + CSVWriter << "\r\n" << ProjectId << ", " << OplogId << ", " << KeyString; + } +}; + +////////////////////////////////////////////////////////////////////////// + +namespace { + + void CbWriteOp(CidStore& CidStore, + bool Details, + bool OpDetails, + bool AttachmentDetails, + ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op, + CbObjectWriter& CbWriter) + { + CbWriter.BeginObject(); + { + CbWriter.AddObjectId("key", Key); + if (Details) + { + CbWriter.AddInteger("lsn", LSN.Number); + CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Op.GetSize())); + } + if (AttachmentDetails) + { + CbWriter.BeginArray("attachments"); + Op.IterateAttachments([&CidStore, &CbWriter](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + CbWriter.BeginObject(); + { + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + CbWriter.AddString("cid", AttachmentHash.ToHexString()); + CbWriter.AddInteger("size", gsl::narrow<uint64_t>(Attachment.GetSize())); + } + CbWriter.EndObject(); + }); + CbWriter.EndArray(); + } + else if (Details) + { + uint64_t AttachmentCount = 0; + size_t AttachmentsSize = 0; + Op.IterateAttachments([&CidStore, &AttachmentCount, &AttachmentsSize](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + AttachmentCount++; + IoBuffer Attachment = CidStore.FindChunkByCid(AttachmentHash); + AttachmentsSize += Attachment.GetSize(); + }); + if (AttachmentCount > 0) + { + CbWriter.AddInteger("attachments", AttachmentCount); + CbWriter.AddInteger("attachmentssize", gsl::narrow<uint64_t>(AttachmentsSize)); + } + } + if (OpDetails) + { + CbWriter.BeginObject("op"); + for (const CbFieldView& Field : Op) + { + if (!Field.HasName()) + { + CbWriter.AddField(Field); + continue; + } + std::string_view FieldName = Field.GetName(); + CbWriter.AddField(FieldName, Field); + } + CbWriter.EndObject(); + } + } + CbWriter.EndObject(); + }; + + void CbWriteOplogOps(CidStore& CidStore, + ProjectStore::Oplog& Oplog, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginArray("ops"); + { + Oplog.IterateOplogWithKey([&Cbo, &CidStore, Details, OpDetails, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CbWriteOp(CidStore, Details, OpDetails, AttachmentDetails, LSN, Key, Op, Cbo); + }); + } + Cbo.EndArray(); + } + + void CbWriteOplog(CidStore& CidStore, + ProjectStore::Oplog& Oplog, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginObject(); + { + Cbo.AddString("name", Oplog.OplogId()); + CbWriteOplogOps(CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndObject(); + } + + void CbWriteOplogs(CidStore& CidStore, + ProjectStore::Project& Project, + std::vector<std::string> OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginArray("oplogs"); + { + for (const std::string& OpLogId : OpLogs) + { + Ref<ProjectStore::Oplog> Oplog = Project.OpenOplog(OpLogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (Oplog) + { + CbWriteOplog(CidStore, *Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + } + } + Cbo.EndArray(); + } + + void CbWriteProject(CidStore& CidStore, + ProjectStore::Project& Project, + std::vector<std::string> OpLogs, + bool Details, + bool OpDetails, + bool AttachmentDetails, + CbObjectWriter& Cbo) + { + Cbo.BeginObject(); + { + Cbo.AddString("name", Project.Identifier); + CbWriteOplogs(CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndObject(); + } + + struct CreateRemoteStoreResult + { + std::shared_ptr<RemoteProjectStore> Store; + std::string Description; + }; + + CreateRemoteStoreResult CreateRemoteStore(CbObjectView Params, + AuthMgr& AuthManager, + size_t MaxBlockSize, + size_t MaxChunkEmbedSize, + const std::filesystem::path& TempFilePath) + { + ZEN_MEMSCOPE(GetProjectHttpTag()); + + using namespace std::literals; + + std::shared_ptr<RemoteProjectStore> RemoteStore; + + if (CbObjectView File = Params["file"sv].AsObjectView(); File) + { + std::filesystem::path FolderPath(File["path"sv].AsString()); + if (FolderPath.empty()) + { + return {nullptr, "Missing file path"}; + } + std::string_view Name(File["name"sv].AsString()); + if (Name.empty()) + { + return {nullptr, "Missing file name"}; + } + std::string_view OptionalBaseName(File["basename"sv].AsString()); + bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); + bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); + + FileRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + FolderPath, + std::string(Name), + std::string(OptionalBaseName), + ForceDisableBlocks, + ForceEnableTempBlocks}; + RemoteStore = CreateFileRemoteStore(Options); + } + + if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) + { + std::string_view CloudServiceUrl = Cloud["url"sv].AsString(); + if (CloudServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(CloudServiceUrl); + std::string_view Namespace = Cloud["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Cloud["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Cloud["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Cloud["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Cloud["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Cloud["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view KeyParam = Cloud["key"sv].AsString(); + if (KeyParam.empty()) + { + return {nullptr, "Missing key"}; + } + if (KeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid key"}; + } + IoHash Key = IoHash::FromHexString(KeyParam); + if (Key == IoHash::Zero) + { + return {nullptr, "Invalid key string"}; + } + IoHash BaseKey = IoHash::Zero; + std::string_view BaseKeyParam = Cloud["basekey"sv].AsString(); + if (!BaseKeyParam.empty()) + { + if (BaseKeyParam.length() != IoHash::StringLength) + { + return {nullptr, "Invalid base key"}; + } + BaseKey = IoHash::FromHexString(BaseKeyParam); + if (BaseKey == IoHash::Zero) + { + return {nullptr, "Invalid base key string"}; + } + } + + bool ForceDisableBlocks = Cloud["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); + + JupiterRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + Key, + BaseKey, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2}; + RemoteStore = CreateJupiterRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) + { + std::string_view Url = Zen["url"sv].AsString(); + std::string_view Project = Zen["project"sv].AsString(); + if (Project.empty()) + { + return {nullptr, "Missing project"}; + } + std::string_view Oplog = Zen["oplog"sv].AsString(); + if (Oplog.empty()) + { + return {nullptr, "Missing oplog"}; + } + ZenRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + std::string(Url), + std::string(Project), + std::string(Oplog)}; + RemoteStore = CreateZenRemoteStore(Options, TempFilePath); + } + + if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) + { + std::string_view BuildsServiceUrl = Builds["url"sv].AsString(); + if (BuildsServiceUrl.empty()) + { + return {nullptr, "Missing service url"}; + } + + std::string Url = UrlDecode(BuildsServiceUrl); + std::string_view Namespace = Builds["namespace"sv].AsString(); + if (Namespace.empty()) + { + return {nullptr, "Missing namespace"}; + } + std::string_view Bucket = Builds["bucket"sv].AsString(); + if (Bucket.empty()) + { + return {nullptr, "Missing bucket"}; + } + std::string_view OpenIdProvider = Builds["openid-provider"sv].AsString(); + std::string AccessToken = std::string(Builds["access-token"sv].AsString()); + if (AccessToken.empty()) + { + std::string_view AccessTokenEnvVariable = Builds["access-token-env"].AsString(); + if (!AccessTokenEnvVariable.empty()) + { + AccessToken = GetEnvVariable(AccessTokenEnvVariable); + } + } + std::filesystem::path OidcExePath; + if (std::string_view OidcExePathString = Builds["oidc-exe-path"].AsString(); !OidcExePathString.empty()) + { + std::filesystem::path OidcExePathMaybe(OidcExePathString); + if (IsFile(OidcExePathMaybe)) + { + OidcExePath = std::move(OidcExePathMaybe); + } + else + { + ZEN_WARN("Path to OidcToken executable '{}' can not be reached by server", OidcExePathString); + } + } + std::string_view BuildIdParam = Builds["buildsid"sv].AsString(); + if (BuildIdParam.empty()) + { + return {nullptr, "Missing build id"}; + } + if (BuildIdParam.length() != Oid::StringLength) + { + return {nullptr, "Invalid build id"}; + } + Oid BuildId = Oid::FromHexString(BuildIdParam); + if (BuildId == Oid::Zero) + { + return {nullptr, "Invalid build id string"}; + } + + bool ForceDisableBlocks = Builds["disableblocks"sv].AsBool(false); + bool ForceDisableTempBlocks = Builds["disabletempblocks"sv].AsBool(false); + bool AssumeHttp2 = Builds["assumehttp2"sv].AsBool(false); + + MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); + IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); + + BuildsRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + BuildId, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2, + MetaData}; + RemoteStore = CreateJupiterBuildsRemoteStore(Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + + if (!RemoteStore) + { + return {nullptr, "Unknown remote store type"}; + } + + return {std::move(RemoteStore), ""}; + } + + std::pair<HttpResponseCode, std::string> ConvertResult(const RemoteProjectStore::Result& Result) + { + if (Result.ErrorCode == 0) + { + return {HttpResponseCode::OK, Result.Text}; + } + return {static_cast<HttpResponseCode>(Result.ErrorCode), + Result.Reason.empty() ? Result.Text + : Result.Text.empty() ? Result.Reason + : fmt::format("{}: {}", Result.Reason, Result.Text)}; + } + +} // namespace + +////////////////////////////////////////////////////////////////////////// + +HttpProjectService::HttpProjectService(CidStore& Store, + ProjectStore* Projects, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue) +: m_Log(logging::Get("project")) +, m_CidStore(Store) +, m_ProjectStore(Projects) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_AuthMgr(AuthMgr) +, m_OpenProcessCache(InOpenProcessCache) +, m_JobQueue(InJobQueue) +{ + ZEN_MEMSCOPE(GetProjectHttpTag()); + + using namespace std::literals; + + m_Router.AddPattern("project", "([[:alnum:]_.]+)"); + m_Router.AddPattern("log", "([[:alnum:]_.]+)"); + m_Router.AddPattern("op", "([[:digit:]]+?)"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + m_Router.AddPattern("hash", "([[:xdigit:]]{40})"); + + m_Router.RegisterRoute( + "", + [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "list", + [this](HttpRouterRequest& Req) { HandleProjectListRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/batch", + [this](HttpRouterRequest& Req) { HandleChunkBatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/files", + [this](HttpRouterRequest& Req) { HandleFilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/chunkinfos", + [this](HttpRouterRequest& Req) { HandleChunkInfosRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}/info", + [this](HttpRouterRequest& Req) { HandleChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{chunk}", + [this](HttpRouterRequest& Req) { HandleChunkByIdRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{hash}", + [this](HttpRouterRequest& Req) { HandleChunkByCidRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/prep", + [this](HttpRouterRequest& Req) { HandleOplogOpPrepRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/new", + [this](HttpRouterRequest& Req) { HandleOplogOpNewRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/validate", + [this](HttpRouterRequest& Req) { HandleOplogValidateRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/{op}", + [this](HttpRouterRequest& Req) { HandleOpLogOpRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}/oplog/{log}", + [this](HttpRouterRequest& Req) { HandleOpLogRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{project}/oplog/{log}/entries", + [this](HttpRouterRequest& Req) { HandleOpLogEntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{project}", + [this](HttpRouterRequest& Req) { HandleProjectRequest(Req); }, + HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kPost | HttpVerb::kDelete); + + // Push a oplog container + m_Router.RegisterRoute( + "{project}/oplog/{log}/save", + [this](HttpRouterRequest& Req) { HandleOplogSaveRequest(Req); }, + HttpVerb::kPost); + + // Pull a oplog container + m_Router.RegisterRoute( + "{project}/oplog/{log}/load", + [this](HttpRouterRequest& Req) { HandleOplogLoadRequest(Req); }, + HttpVerb::kGet); + + // Do an rpc style operation on project/oplog + m_Router.RegisterRoute( + "{project}/oplog/{log}/rpc", + [this](HttpRouterRequest& Req) { HandleRpcRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "details\\$", + [this](HttpRouterRequest& Req) { HandleDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}", + [this](HttpRouterRequest& Req) { HandleProjectDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}/{log}", + [this](HttpRouterRequest& Req) { HandleOplogDetailsRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "details\\$/{project}/{log}/{chunk}", + [this](HttpRouterRequest& Req) { HandleOplogOpDetailsRequest(Req); }, + HttpVerb::kGet); + + m_StatusService.RegisterHandler("prj", *this); + m_StatsService.RegisterHandler("prj", *this); +} + +HttpProjectService::~HttpProjectService() +{ + m_StatsService.UnregisterHandler("prj", *this); + m_StatusService.UnregisterHandler("prj", *this); +} + +const char* +HttpProjectService::BaseUri() const +{ + return "/prj/"; +} + +void +HttpProjectService::HandleRequest(HttpServerRequest& Request) +{ + m_ProjectStats.RequestCount++; + + ZEN_MEMSCOPE(GetProjectHttpTag()); + + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + m_ProjectStats.BadRequestCount++; + ZEN_WARN("No route found for {0}", Request.RelativeUri()); + } +} + +void +HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("ProjectService::Stats"); + + const GcStorageSize StoreSize = m_ProjectStore->StorageSize(); + const CidStoreSize CidSize = m_CidStore.TotalSize(); + + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("store"); + { + Cbo.BeginObject("size"); + { + Cbo << "disk" << StoreSize.DiskSize; + Cbo << "memory" << StoreSize.MemorySize; + } + Cbo.EndObject(); + + Cbo.BeginObject("project"); + { + Cbo << "readcount" << m_ProjectStats.ProjectReadCount << "writecount" << m_ProjectStats.ProjectWriteCount << "deletecount" + << m_ProjectStats.ProjectDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("oplog"); + { + Cbo << "readcount" << m_ProjectStats.OpLogReadCount << "writecount" << m_ProjectStats.OpLogWriteCount << "deletecount" + << m_ProjectStats.OpLogDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("op"); + { + Cbo << "hitcount" << m_ProjectStats.OpHitCount << "misscount" << m_ProjectStats.OpMissCount << "writecount" + << m_ProjectStats.OpWriteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_ProjectStats.ChunkHitCount << "misscount" << m_ProjectStats.ChunkMissCount << "writecount" + << m_ProjectStats.ChunkWriteCount; + } + Cbo.EndObject(); + + Cbo << "requestcount" << m_ProjectStats.RequestCount; + Cbo << "badrequestcount" << m_ProjectStats.BadRequestCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("cid"); + { + Cbo.BeginObject("size"); + { + Cbo << "tiny" << CidSize.TinySize; + Cbo << "small" << CidSize.SmallSize; + Cbo << "large" << CidSize.LargeSize; + Cbo << "total" << CidSize.TotalSize; + } + Cbo.EndObject(); + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpProjectService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpProjectService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpProjectService::HandleProjectListRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ProjectList"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + CbArray ProjectsList = m_ProjectStore->GetProjectsList(); + HttpReq.WriteResponse(HttpResponseCode::OK, ProjectsList); +} + +void +HttpProjectService::HandleChunkBatchRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkBatch"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + // Parse Request + + IoBuffer Payload = HttpReq.ReadPayload(); + BinaryReader Reader(Payload); + + struct RequestHeader + { + enum + { + kMagic = 0xAAAA'77AC + }; + uint32_t Magic; + uint32_t ChunkCount; + uint32_t Reserved1; + uint32_t Reserved2; + }; + + struct RequestChunkEntry + { + Oid ChunkId; + uint32_t CorrelationId; + uint64_t Offset; + uint64_t RequestBytes; + }; + + if (Payload.Size() <= sizeof(RequestHeader)) + { + m_ProjectStats.BadRequestCount++; + HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + + RequestHeader RequestHdr; + Reader.Read(&RequestHdr, sizeof RequestHdr); + + if (RequestHdr.Magic != RequestHeader::kMagic) + { + m_ProjectStats.BadRequestCount++; + HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + + std::vector<RequestChunkEntry> RequestedChunks; + RequestedChunks.resize(RequestHdr.ChunkCount); + Reader.Read(RequestedChunks.data(), sizeof(RequestChunkEntry) * RequestHdr.ChunkCount); + + // Make Response + + struct ResponseHeader + { + uint32_t Magic = 0xbada'b00f; + uint32_t ChunkCount; + uint32_t Reserved1 = 0; + uint32_t Reserved2 = 0; + }; + + struct ResponseChunkEntry + { + uint32_t CorrelationId; + uint32_t Flags = 0; + uint64_t ChunkSize; + }; + + std::vector<IoBuffer> OutBlobs; + OutBlobs.emplace_back(sizeof(ResponseHeader) + RequestHdr.ChunkCount * sizeof(ResponseChunkEntry)); + for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) + { + const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + IoBuffer FoundChunk = FoundLog->FindChunk(Project->RootDir, RequestedChunk.ChunkId, nullptr); + if (FoundChunk) + { + if (RequestedChunk.Offset > 0 || RequestedChunk.RequestBytes < uint64_t(-1)) + { + uint64_t Offset = RequestedChunk.Offset; + if (Offset > FoundChunk.Size()) + { + Offset = FoundChunk.Size(); + } + uint64_t Size = RequestedChunk.RequestBytes; + if ((Offset + Size) > FoundChunk.Size()) + { + Size = FoundChunk.Size() - Offset; + } + FoundChunk = IoBuffer(FoundChunk, Offset, Size); + } + } + OutBlobs.emplace_back(std::move(FoundChunk)); + } + uint8_t* ResponsePtr = reinterpret_cast<uint8_t*>(OutBlobs[0].MutableData()); + ResponseHeader ResponseHdr; + ResponseHdr.ChunkCount = RequestHdr.ChunkCount; + memcpy(ResponsePtr, &ResponseHdr, sizeof(ResponseHdr)); + ResponsePtr += sizeof(ResponseHdr); + for (uint32_t ChunkIndex = 0; ChunkIndex < RequestHdr.ChunkCount; ++ChunkIndex) + { + const RequestChunkEntry& RequestedChunk = RequestedChunks[ChunkIndex]; + const IoBuffer& FoundChunk(OutBlobs[ChunkIndex + 1]); + ResponseChunkEntry ResponseChunk; + ResponseChunk.CorrelationId = RequestedChunk.CorrelationId; + if (FoundChunk) + { + ResponseChunk.ChunkSize = FoundChunk.Size(); + m_ProjectStats.ChunkHitCount++; + } + else + { + ResponseChunk.ChunkSize = uint64_t(-1); + m_ProjectStats.ChunkMissCount++; + } + memcpy(ResponsePtr, &ResponseChunk, sizeof(ResponseChunk)); + ResponsePtr += sizeof(ResponseChunk); + } + std::erase_if(OutBlobs, [](IoBuffer Buffer) -> bool { return !Buffer; }); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, OutBlobs); +} + +void +HttpProjectService::HandleFilesRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Files"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + // File manifest fetch, returns the client file list + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = Params.GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Project files for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectFiles(Log(), *Project, *FoundLog, WantedFieldNames); + + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } +} + +void +HttpProjectService::HandleChunkInfosRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkInfos"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + WantedFieldNames.insert("id"); + WantedFieldNames.insert("rawhash"); + WantedFieldNames.insert("rawsize"); + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk infos for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbObject ResponsePayload = ProjectStore::GetProjectChunkInfos(Log(), *Project, *FoundLog, WantedFieldNames); + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(ResponsePayload.GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } +} + +void +HttpProjectService::HandleChunkInfoRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkInfo"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, ChunkId)); + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + CbObject ResponsePayload = ProjectStore::GetChunkInfo(Log(), *Project, *FoundLog, Obj); + if (ResponsePayload) + { + m_ProjectStats.ChunkHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + else + { + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk info for unknown chunk '{}/{}/{}'", ProjectId, OplogId, ChunkId)); + } +} + +void +HttpProjectService::HandleChunkByIdRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkById"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& ChunkId = Req.GetCapture(3); + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + + auto QueryParms = HttpReq.GetQueryParams(); + + if (auto OffsetParm = QueryParms.GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + } + + if (auto SizeParm = QueryParms.GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt<uint64_t>(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (ChunkId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, ChunkId)); + } + + const Oid Obj = Oid::FromHexString(ChunkId); + + HttpContentType AcceptType = HttpReq.AcceptContentType(); + + ProjectStore::GetChunkRangeResult Result = + ProjectStore::GetChunkRange(Log(), *Project, *FoundLog, Obj, Offset, Size, AcceptType, /*OptionalInOutModificationTag*/ nullptr); + + switch (Result.Error) + { + case ProjectStore::GetChunkRangeResult::EError::Ok: + m_ProjectStats.ChunkHitCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' '{}'", ProjectId, OplogId, ChunkId, ToString(Result.ContentType)); + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::NotFound: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Result.ContentType, Result.Chunk); + case ProjectStore::GetChunkRangeResult::EError::MalformedContent: + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + case ProjectStore::GetChunkRangeResult::EError::OutOfRange: + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' OUT OF RANGE", ProjectId, OplogId, ChunkId); + return HttpReq.WriteResponse( + HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Get chunk {}/{}/{} failed. Reason: {}", ProjectId, OplogId, ChunkId, Result.ErrorDescription)); + default: + ZEN_ASSERT(false); + break; + } +} + +void +HttpProjectService::HandleChunkByCidRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ChunkByCid"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& Cid = Req.GetCapture(3); + HttpContentType AcceptType = HttpReq.AcceptContentType(); + HttpContentType RequestType = HttpReq.RequestContentType(); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Chunk request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + if (Cid.length() != IoHash::StringLength) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for invalid chunk id '{}/{}/{}'", ProjectId, OplogId, Cid)); + } + + const IoHash Hash = IoHash::FromHexString(Cid); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + IoBuffer Value = m_ProjectStore->GetChunk(*Project, *FoundLog, Hash); + if (Value) + { + if (AcceptType == ZenContentType::kUnknownContentType || AcceptType == ZenContentType::kBinary || + AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Value)); + IoBuffer DecompressedBuffer = Compressed.Decompress().AsIoBuffer(); + + if (DecompressedBuffer) + { + if (AcceptType == ZenContentType::kJSON || AcceptType == ZenContentType::kYAML || + AcceptType == ZenContentType::kCbObject) + { + CbValidateError CbErr = ValidateCompactBinary(DecompressedBuffer.GetView(), CbValidateMode::Default); + if (!!CbErr) + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG( + "chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not convert to object`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not convert to object", + ToString(AcceptType))); + } + + m_ProjectStats.ChunkHitCount++; + CbObject ContainerObject = LoadCompactBinaryObject(DecompressedBuffer); + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerObject); + } + else + { + Value = DecompressedBuffer; + Value.SetContentType(ZenContentType::kBinary); + } + } + else + { + m_ProjectStats.BadRequestCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' WRONGTYPE. Reason: `Requested {} format, but could not decompress stored data`", + ProjectId, + OplogId, + Cid, + ToString(AcceptType)); + return HttpReq.WriteResponse( + HttpResponseCode::NotAcceptable, + HttpContentType::kText, + fmt::format("Content format not supported, requested {} format, but could not decompress stored data", + ToString(AcceptType))); + } + } + m_ProjectStats.ChunkHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK, Value.GetContentType(), Value); + } + else + { + m_ProjectStats.ChunkMissCount++; + ZEN_DEBUG("chunk - '{}/{}/{}' MISSING", ProjectId, OplogId, Cid); + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + } + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + if (RequestType != HttpContentType::kCompressedBinary) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk request for chunk id '{}/{}'/'{}' as unexpected content type: '{}'", + ProjectId, + OplogId, + Cid, + ToString(RequestType))); + } + IoBuffer Payload = HttpReq.ReadPayload(); + Payload.SetContentType(RequestType); + bool IsNew = m_ProjectStore->PutChunk(*Project, *FoundLog, Hash, std::move(Payload)); + + m_ProjectStats.ChunkWriteCount++; + return HttpReq.WriteResponse(IsNew ? HttpResponseCode::Created : HttpResponseCode::OK); + } + break; + } +} + +void +HttpProjectService::HandleOplogOpPrepRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpPrep"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + // This operation takes a list of referenced hashes and decides which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + CbValidateError ValidateResult; + if (CbObject RequestObject = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::vector<IoHash> NeedList; + + { + eastl::fixed_vector<IoHash, 16> ChunkList; + CbArrayView HaveList = RequestObject["have"sv].AsArrayView(); + ChunkList.reserve(HaveList.Num()); + for (auto& Entry : HaveList) + { + ChunkList.push_back(Entry.AsHash()); + } + + NeedList = FoundLog->CheckPendingChunkReferences(std::span(begin(ChunkList), end(ChunkList)), std::chrono::minutes(2)); + } + + CbObjectWriter Cbo(1 + 1 + 5 + NeedList.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : NeedList) + { + ZEN_DEBUG("prep - NEED: {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Response); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid compact binary format: '{}'", ToString(ValidateResult))); + } +} + +void +HttpProjectService::HandleOplogOpNewRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpNew"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + bool IsUsingSalt = false; + IoHash SaltHash = IoHash::Zero; + + if (std::string_view SaltParam = Params.GetValue("salt"sv); SaltParam.empty() == false) + { + const uint32_t Salt = std::stoi(std::string(SaltParam)); + SaltHash = IoHash::HashBuffer(&Salt, sizeof Salt); + IsUsingSalt = true; + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + IoBuffer Payload = HttpReq.ReadPayload(); + + // This will attempt to open files which may not exist for the case where + // the prep step rejected the chunk. This should be fixed since there's + // a performance cost associated with any file system activity + + bool IsValid = true; + std::vector<IoHash> MissingChunks; + + CbPackage::AttachmentResolver Resolver = [&](const IoHash& Hash) -> SharedBuffer { + if (m_CidStore.ContainsChunk(Hash)) + { + // Return null attachment as we already have it, no point in reading it and storing it again + return {}; + } + + IoHash AttachmentId; + if (IsUsingSalt) + { + IoHash AttachmentSpec[]{SaltHash, Hash}; + AttachmentId = IoHash::HashBuffer(MakeMemoryView(AttachmentSpec)); + } + else + { + AttachmentId = Hash; + } + + std::filesystem::path AttachmentPath = Oplog.TempPath() / AttachmentId.ToHexString(); + if (IoBuffer Data = IoBufferBuilder::MakeFromTemporaryFile(AttachmentPath)) + { + Data.SetDeleteOnClose(true); + return SharedBuffer(std::move(Data)); + } + else + { + IsValid = false; + MissingChunks.push_back(Hash); + + return {}; + } + }; + + CbPackage Package; + + if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) + { + CbValidateError ValidateResult; + if (CbObject Core = ValidateAndReadCompactBinaryObject(IoBuffer(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Core) + { + Package.SetObject(Core); + } + else + { + std::filesystem::path BadPackagePath = + Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); + + ZEN_WARN("Received malformed package ('{}')! Saving payload to '{}'", ToString(ValidateResult), BadPackagePath); + + WriteFile(BadPackagePath, Payload); + + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + u8"request body must be a compact binary object or package in legacy format"); + } + } + + m_ProjectStats.ChunkMissCount += MissingChunks.size(); + + if (!IsValid) + { + ExtendableStringBuilder<256> ResponseText; + ResponseText.Append("Missing chunk references: "); + + bool IsFirst = true; + for (const auto& Hash : MissingChunks) + { + if (IsFirst) + { + IsFirst = false; + } + else + { + ResponseText.Append(", "); + } + Hash.ToHexString(ResponseText); + } + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, ResponseText); + } + + CbObject Core = Package.GetObject(); + + if (!Core["key"sv]) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "No oplog entry key specified"); + } + + eastl::fixed_vector<IoHash, 16> ReferencedChunks; + Core.IterateAttachments([&ReferencedChunks](CbFieldView View) { ReferencedChunks.push_back(View.AsAttachment()); }); + + // Write core to oplog + + size_t AttachmentCount = Package.GetAttachments().size(); + const ProjectStore::LogSequenceNumber OpLsn = Oplog.AppendNewOplogEntry(Package); + if (!OpLsn) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + m_ProjectStats.ChunkWriteCount += AttachmentCount; + + // Once we stored the op, we no longer need to retain any chunks this op references + if (!ReferencedChunks.empty()) + { + FoundLog->RemovePendingChunkReferences(std::span(begin(ReferencedChunks), end(ReferencedChunks))); + } + + m_ProjectStats.OpWriteCount++; + ZEN_DEBUG("'{}/{}' op #{} ({}) - '{}'", ProjectId, OplogId, OpLsn.Number, NiceBytes(Payload.Size()), Core["key"sv].AsString()); + HttpReq.WriteResponse(HttpResponseCode::Created); +} + +void +HttpProjectService::HandleOplogValidateRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpValidate"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ZenContentType::kText, fmt::format("Project '{}' not found", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + ZenContentType::kText, + fmt::format("Oplog '{}' not found in project '{}'", OplogId, ProjectId)); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + std::atomic_bool CancelFlag = false; + ProjectStore::Oplog::ValidationResult Result = Oplog.Validate(Project->RootDir, CancelFlag, &GetSmallWorkerPool(EWorkloadType::Burst)); + tsl::robin_map<Oid, std::string, Oid::Hasher> KeyNameLookup; + KeyNameLookup.reserve(Result.OpKeys.size()); + for (const auto& It : Result.OpKeys) + { + KeyNameLookup.insert_or_assign(It.first, It.second); + } + CbObjectWriter Writer; + Writer << "HasMissingData" << !Result.IsEmpty(); + Writer << "OpCount" << Result.OpCount; + Writer << "LSNLow" << Result.LSNLow.Number; + Writer << "LSNHigh" << Result.LSNHigh.Number; + if (!Result.MissingFiles.empty()) + { + Writer.BeginArray("MissingFiles"); + for (const auto& MissingFile : Result.MissingFiles) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingFile.first; + Writer << "KeyName" << KeyNameLookup[MissingFile.first]; + Writer << "Id" << MissingFile.second.Id; + Writer << "Hash" << MissingFile.second.Hash; + Writer << "ServerPath" << MissingFile.second.ServerPath; + Writer << "ClientPath" << MissingFile.second.ClientPath; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingChunks.empty()) + { + Writer.BeginArray("MissingChunks"); + for (const auto& MissingChunk : Result.MissingChunks) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingChunk.first; + Writer << "KeyName" << KeyNameLookup[MissingChunk.first]; + Writer << "Id" << MissingChunk.second.Id; + Writer << "Hash" << MissingChunk.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingMetas.empty()) + { + Writer.BeginArray("MissingMetas"); + for (const auto& MissingMeta : Result.MissingMetas) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Id" << MissingMeta.second.Id; + Writer << "Hash" << MissingMeta.second.Hash; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + if (!Result.MissingAttachments.empty()) + { + Writer.BeginArray("MissingAttachments"); + for (const auto& MissingMeta : Result.MissingAttachments) + { + Writer.BeginObject(); + { + Writer << "Key" << MissingMeta.first; + Writer << "KeyName" << KeyNameLookup[MissingMeta.first]; + Writer << "Hash" << MissingMeta.second; + } + Writer.EndObject(); + } + Writer.EndArray(); + } + CbObject Response = Writer.Save(); + HttpReq.WriteResponse(HttpResponseCode::OK, Response); +} + +void +HttpProjectService::HandleOpLogOpRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOp"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const std::string_view ProjectId = Req.GetCapture(1); + const std::string_view OplogId = Req.GetCapture(2); + const std::string_view OpIdString = Req.GetCapture(3); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ false); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + ProjectStore::Oplog& Oplog = *FoundLog; + + if (const std::optional<int32_t> OpId = ParseInt<uint32_t>(OpIdString)) + { + if (std::optional<CbObject> MaybeOp = Oplog.GetOpByIndex(ProjectStore::LogSequenceNumber(OpId.value()))) + { + CbObject& Op = MaybeOp.value(); + if (HttpReq.AcceptContentType() == ZenContentType::kCbPackage) + { + CbPackage Package; + Package.SetObject(Op); + + Op.IterateAttachments([&](CbFieldView FieldView) { + const IoHash AttachmentHash = FieldView.AsAttachment(); + IoBuffer Payload = m_CidStore.FindChunkByCid(AttachmentHash); + if (Payload) + { + switch (Payload.GetContentType()) + { + case ZenContentType::kCbObject: + { + CbValidateError ValidateResult; + if (CbObject Object = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && Object) + { + Package.AddAttachment(CbAttachment(Object)); + } + else + { + // Error - malformed object + ZEN_WARN("malformed object returned for {} ('{}')", AttachmentHash, ToString(ValidateResult)); + } + } + break; + + case ZenContentType::kCompressedBinary: + if (CompressedBuffer Compressed = CompressedBuffer::FromCompressedNoValidate(std::move(Payload))) + { + Package.AddAttachment(CbAttachment(Compressed, AttachmentHash)); + } + else + { + // Error - not compressed! + + ZEN_WARN("invalid compressed binary returned for {}", AttachmentHash); + } + break; + + default: + Package.AddAttachment(CbAttachment(SharedBuffer(Payload))); + break; + } + } + }); + m_ProjectStats.OpHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::Accepted, Package); + } + else + { + // Client cannot accept a package, so we only send the core object + m_ProjectStats.OpHitCount++; + return HttpReq.WriteResponse(HttpResponseCode::Accepted, Op); + } + } + } + m_ProjectStats.OpMissCount++; + return HttpReq.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpProjectService::HandleOpLogRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Oplog"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + using namespace std::literals; + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, fmt::format("project {} not found", ProjectId)); + } + Project->TouchProject(); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + Ref<ProjectStore::Oplog> OplogIt = Project->ReadOplog(OplogId); + if (!OplogIt) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("oplog {} not found in project {}", OplogId, ProjectId)); + } + + ProjectStore::Oplog& Log = *OplogIt; + + CbObjectWriter Cb; + Cb << "id"sv << Log.OplogId() << "project"sv << Project->Identifier << "tempdir"sv << Log.TempPath().c_str() + << "markerpath"sv << Log.MarkerPath().c_str() << "totalsize"sv << Log.TotalSize() << "opcount" << Log.OplogCount() + << "expired"sv << Project->IsExpired(GcClock::TimePoint::min(), Log); + HttpReq.WriteResponse(HttpResponseCode::OK, Cb.Save()); + + m_ProjectStats.OpLogReadCount++; + } + break; + + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + std::filesystem::path OplogMarkerPath; + if (CbObject Params = HttpReq.ReadPayloadObject()) + { + OplogMarkerPath = Params["gcpath"sv].AsString(); + } + + Ref<ProjectStore::Oplog> OplogIt = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!OplogIt) + { + if (!Project->NewOplog(OplogId, OplogMarkerPath)) + { + // TODO: indicate why the operation failed! + return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); + } + Project->TouchOplog(OplogId); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::Created); + } + + // I guess this should ultimately be used to execute RPCs but for now, it + // does absolutely nothing + + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + break; + + case HttpVerb::kPut: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + std::filesystem::path OplogMarkerPath; + if (CbObject Params = HttpReq.ReadPayloadObject()) + { + OplogMarkerPath = Params["gcpath"sv].AsString(); + } + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + if (!Project->NewOplog(OplogId, OplogMarkerPath)) + { + // TODO: indicate why the operation failed! + return HttpReq.WriteResponse(HttpResponseCode::InternalServerError); + } + Project->TouchOplog(OplogId); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("established oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::Created); + } + Project->TouchOplog(OplogId); + + FoundLog->Update(OplogMarkerPath); + + m_ProjectStats.OpLogWriteCount++; + ZEN_INFO("updated oplog '{}/{}', gc marker file at '{}'", ProjectId, OplogId, OplogMarkerPath); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + break; + + case HttpVerb::kDelete: + { + ZEN_INFO("deleting oplog '{}/{}'", ProjectId, OplogId); + + if (Project->DeleteOplog(OplogId)) + { + m_ProjectStats.OpLogDeleteCount++; + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::Locked, + HttpContentType::kText, + fmt::format("oplog {}/{} is in use", ProjectId, OplogId)); + } + } + break; + + default: + break; + } +} + +std::optional<OplogReferencedSet> +LoadReferencedSet(ProjectStore::Project& Project, ProjectStore::Oplog& Log) +{ + using namespace std::literals; + + Oid ReferencedSetOplogId = OpKeyStringAsOid(OplogReferencedSet::ReferencedSetOplogKey); + std::optional<CbObject> ReferencedSetOp = Log.GetOpByKey(ReferencedSetOplogId); + if (!ReferencedSetOp) + { + return std::optional<OplogReferencedSet>(); + } + // We expect only a single file in the "files" array; get the chunk for the first file + CbFieldView FileField = *(*ReferencedSetOp)["files"sv].AsArrayView().CreateViewIterator(); + Oid ChunkId = FileField.AsObjectView()["id"sv].AsObjectId(); + if (ChunkId == Oid::Zero) + { + return std::optional<OplogReferencedSet>(); + } + + return OplogReferencedSet::LoadFromChunk(Log.FindChunk(Project.RootDir, ChunkId, nullptr)); +} + +void +HttpProjectService::HandleOpLogEntriesRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogEntries"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> FoundLog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + Project->TouchOplog(OplogId); + + CbObjectWriter Response; + + if (FoundLog->OplogCount() > 0) + { + std::unordered_set<std::string> FieldNamesFilter; + auto FilterObject = [&FieldNamesFilter](CbObjectView& Object) -> CbObject { + CbObject RewrittenOp = RewriteCbObject(Object, [&FieldNamesFilter](CbObjectWriter&, CbFieldView Field) -> bool { + if (FieldNamesFilter.contains(std::string(Field.GetName()))) + { + return false; + } + + return true; + }); + + return RewrittenOp; + }; + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + if (auto FieldFilter = HttpServerRequest::Decode(Params.GetValue("fieldfilter")); !FieldFilter.empty()) + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + FieldNamesFilter.insert(std::string(FieldName)); + return true; + }); + } + + if (auto OpKey = Params.GetValue("opkey"); !OpKey.empty()) + { + Oid OpKeyId = OpKeyStringAsOid(OpKey); + std::optional<CbObject> Op = FoundLog->GetOpByKey(OpKeyId); + + if (Op.has_value()) + { + if (FieldNamesFilter.empty()) + { + Response << "entry"sv << Op.value(); + } + else + { + Response << "entry"sv << FilterObject(Op.value()); + } + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + } + else + { + ProjectStore::Oplog::Paging EntryPaging; + if (std::string_view Param = Params.GetValue("start"); !Param.empty()) + { + if (auto Value = ParseInt<int32>(Param)) + { + EntryPaging.Start = *Value; + } + } + if (std::string_view Param = Params.GetValue("count"); !Param.empty()) + { + if (auto Value = ParseInt<int32>(Param)) + { + EntryPaging.Count = *Value; + } + } + + std::optional<OplogReferencedSet> MaybeReferencedSet; + if (auto TrimString = Params.GetValue("trim_by_referencedset"); TrimString == "true") + { + MaybeReferencedSet = LoadReferencedSet(*Project, *FoundLog); + } + Response.BeginArray("entries"sv); + + bool ShouldFilterFields = !FieldNamesFilter.empty(); + + if (MaybeReferencedSet) + { + const OplogReferencedSet& ReferencedSet = MaybeReferencedSet.value(); + FoundLog->IterateOplogWithKey( + [this, &Response, &FilterObject, ShouldFilterFields, &ReferencedSet](ProjectStore::LogSequenceNumber /* LSN */, + const Oid& Key, + CbObjectView Op) { + if (!ReferencedSet.Contains(Key)) + { + if (!OplogReferencedSet::IsNonPackage(Op["key"].AsString())) + { + return; + } + } + + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } + else + { + FoundLog->IterateOplog( + [this, &Response, &FilterObject, ShouldFilterFields](CbObjectView Op) { + if (ShouldFilterFields) + { + Response << FilterObject(Op); + } + else + { + Response << Op; + } + }, + EntryPaging); + } + + Response.EndArray(); + } + } + if (HttpReq.AcceptContentType() == HttpContentType::kCompressedBinary) + { + CompositeBuffer Payload = CompressedBuffer::Compress(Response.Save().GetBuffer()).GetCompressed(); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCompressedBinary, Payload); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + } +} + +void +HttpProjectService::HandleProjectRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Project"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const std::string_view ProjectId = Req.GetCapture(1); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kPost: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + m_ProjectStats.ProjectWriteCount++; + HttpReq.WriteResponse(HttpResponseCode::Created); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } + } + break; + + case HttpVerb::kPut: + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + CbValidateError ValidateResult; + if (CbObject Params = ValidateAndReadCompactBinaryObject(HttpReq.ReadPayload(), ValidateResult); + ValidateResult == CbValidateError::None) + { + std::filesystem::path Root = Params["root"sv].AsU8String(); // Workspace root (i.e `D:/UE5/`) + std::filesystem::path EngineRoot = Params["engine"sv].AsU8String(); // Engine root (i.e `D:/UE5/Engine`) + std::filesystem::path ProjectRoot = + Params["project"sv].AsU8String(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::filesystem::path ProjectFilePath = + Params["projectfile"sv].AsU8String(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) + + if (m_ProjectStore->UpdateProject(ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath)) + { + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("updated project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; + m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); + + m_ProjectStats.ProjectWriteCount++; + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ProjectId, + Root, + EngineRoot, + ProjectRoot, + ProjectFilePath, + ProjectFilePath.empty() ? ", project will not be GCd due to empty project file path" : ""); + + HttpReq.WriteResponse(HttpResponseCode::Created); + } + } + else + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Malformed compact binary object: '{}'", ToString(ValidateResult))); + } + } + break; + + case HttpVerb::kGet: + { + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("project {} not found", ProjectId)); + } + Project->TouchProject(); + + std::vector<std::string> OpLogs = Project->ScanForOplogs(); + + CbObjectWriter Response; + Response << "id"sv << Project->Identifier; + Response << "root"sv << PathToUtf8(Project->RootDir); + Response << "engine"sv << PathToUtf8(Project->EngineRootDir); + Response << "project"sv << PathToUtf8(Project->ProjectRootDir); + Response << "projectfile"sv << PathToUtf8(Project->ProjectFilePath); + + Response.BeginArray("oplogs"sv); + for (const std::string& OplogId : OpLogs) + { + Response.BeginObject(); + Response << "id"sv << OplogId; + Response.EndObject(); + } + Response.EndArray(); // oplogs + + HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); + + m_ProjectStats.ProjectReadCount++; + } + break; + + case HttpVerb::kDelete: + { + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("project {} not found", ProjectId)); + } + + ZEN_INFO("deleting project '{}'", ProjectId); + if (!m_ProjectStore->DeleteProject(ProjectId)) + { + return HttpReq.WriteResponse(HttpResponseCode::Locked, + HttpContentType::kText, + fmt::format("project {} is in use", ProjectId)); + } + + m_ProjectStats.ProjectDeleteCount++; + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + break; + + default: + break; + } +} + +void +HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogSave"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + if (HttpReq.RequestContentType() != HttpContentType::kCbObject) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid content type"); + } + IoBuffer Payload = HttpReq.ReadPayload(); + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ false); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Write oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + CbValidateError ValidateResult; + if (CbObject ContainerObject = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult == CbValidateError::None && ContainerObject) + { + RwLock AttachmentsLock; + tsl::robin_set<IoHash, IoHash::Hasher> Attachments; + + auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); }; + auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector<IoHash>&& ChunkHashes) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + if (BlockHash != IoHash::Zero) + { + Attachments.insert(BlockHash); + } + else + { + Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); + } + }; + auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { + RwLock::ExclusiveLockScope _(AttachmentsLock); + Attachments.insert(RawHash); + }; + + auto OnChunkedAttachment = [](const ChunkedInfo&) {}; + + auto OnReferencedAttachments = [&Oplog](std::span<IoHash> RawHashes) { Oplog->CaptureAddedAttachments(RawHashes); }; + + // Make sure we retain any attachments we download before writing the oplog + Oplog->EnableUpdateCapture(); + auto _ = MakeGuard([&Oplog]() { Oplog->DisableUpdateCapture(); }); + + RemoteProjectStore::Result Result = SaveOplogContainer(*Oplog, + ContainerObject, + OnReferencedAttachments, + HasAttachment, + OnNeedBlock, + OnNeedAttachment, + OnChunkedAttachment, + nullptr); + + if (Result.ErrorCode == 0) + { + if (Attachments.empty()) + { + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + CbObjectWriter Cbo(1 + 1 + 5 + Attachments.size() * (1 + sizeof(IoHash::Hash)) + 1); + Cbo.BeginArray("need"); + { + for (const IoHash& Hash : Attachments) + { + ZEN_DEBUG("Need attachment {}", Hash); + Cbo << Hash; + } + } + Cbo.EndArray(); // "need" + + CbObject ResponsePayload = Cbo.Save(); + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponsePayload); + } + } + else + { + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + Result.ErrorCode, + Result.Reason); + + if (Result.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(Result.ErrorCode), HttpContentType::kText, Result.Reason); + } + } + } + else + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid payload: '{}'", ToString(ValidateResult))); + } +} + +void +HttpProjectService::HandleOplogLoadRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogLoad"); + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + + if (HttpReq.AcceptContentType() != HttpContentType::kCbObject) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid accept content type"); + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ true, /*VerifyPathOnDisk*/ true); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Read oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + size_t MaxBlockSize = RemoteStoreOptions::DefaultMaxBlockSize; + if (auto Param = Params.GetValue("maxblocksize"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + MaxBlockSize = Value.value(); + } + } + size_t MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize; + if (auto Param = Params.GetValue("maxchunkembedsize"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + MaxChunkEmbedSize = Value.value(); + } + } + size_t MaxChunksPerBlock = RemoteStoreOptions::DefaultMaxChunksPerBlock; + if (auto Param = Params.GetValue("maxchunksperblock"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + MaxChunksPerBlock = Value.value(); + } + } + + size_t ChunkFileSizeLimit = RemoteStoreOptions::DefaultChunkFileSizeLimit; + if (auto Param = Params.GetValue("chunkfilesizelimit"); Param.empty() == false) + { + if (auto Value = ParseInt<size_t>(Param)) + { + ChunkFileSizeLimit = Value.value(); + } + } + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::LoadContainerResult ContainerResult = BuildContainer( + m_CidStore, + *Project, + *Oplog, + WorkerPool, + MaxBlockSize, + MaxChunkEmbedSize, + MaxChunksPerBlock, + ChunkFileSizeLimit, + /* BuildBlocks */ false, + /* IgnoreMissingAttachments */ false, + /* AllowChunking*/ false, + [](CompressedBuffer&&, ChunkBlockDescription&&) {}, + [](const IoHash&, TGetAttachmentBufferFunc&&) {}, + [](std::vector<std::pair<IoHash, FetchChunkFunc>>&&) {}, + /* EmbedLooseFiles*/ false); + + if (ContainerResult.ErrorCode == 0) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, ContainerResult.ContainerObject); + } + else + { + ZEN_DEBUG("Request {}: '{}' failed with {}. Reason: `{}`", + ToString(HttpReq.RequestVerb()), + HttpReq.QueryString(), + ContainerResult.ErrorCode, + ContainerResult.Reason); + + if (ContainerResult.Reason.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode)); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode(ContainerResult.ErrorCode), HttpContentType::kText, ContainerResult.Reason); + } + } +} + +void +HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Rpc"); + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + IoBuffer Payload = HttpReq.ReadPayload(); + + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + CbPackage Package; + CbObject Cb; + switch (PayloadContentType) + { + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + } + } + break; + case HttpContentType::kCbObject: + { + CbValidateError ValidateResult; + if (Cb = ValidateAndReadCompactBinaryObject(std::move(Payload), ValidateResult); + ValidateResult != CbValidateError::None || !Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Content format not supported, expected compact binary format ('{}')", ToString(ValidateResult))); + } + break; + } + case HttpContentType::kCbPackage: + try + { + Package = ParsePackageMessage(Payload); + Cb = Package.GetObject(); + } + catch (const std::invalid_argument& ex) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Failed to parse package request, reason: '{}'", ex.what())); + } + if (!Cb) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected package message format"); + } + break; + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + } + + Ref<ProjectStore::Project> Project = m_ProjectStore->OpenProject(ProjectId); + if (!Project) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown project '{}'", ProjectId)); + } + Project->TouchProject(); + + std::string_view Method = Cb["method"sv].AsString(); + + bool VerifyPathOnDisk = Method != "getchunks"sv; + + Ref<ProjectStore::Oplog> Oplog = Project->OpenOplog(OplogId, /*AllowCompact*/ false, VerifyPathOnDisk); + if (!Oplog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Rpc oplog request for unknown oplog '{}/{}'", ProjectId, OplogId)); + } + Project->TouchOplog(OplogId); + + uint32_t MethodHash = HashStringDjb2(Method); + + switch (MethodHash) + { + case HashStringDjb2("import"sv): + { + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool CleanOplog = Params["clean"].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ChunkStore = &m_CidStore, + ActualRemoteStore = std::move(RemoteStore), + Oplog, + Force, + IgnoreMissingAttachments, + CleanOplog](JobContext& Context) { + Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", + Oplog->GetOuterProjectIdentifier(), + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description)); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = LoadOplog(m_CidStore, + *ActualRemoteStore, + *Oplog, + NetworkWorkerPool, + WorkerPool, + Force, + IgnoreMissingAttachments, + CleanOplog, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("export"sv): + { + CbObjectView Params = Cb["params"sv].AsObjectView(); + size_t MaxBlockSize = Params["maxblocksize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxBlockSize); + size_t MaxChunkEmbedSize = Params["maxchunkembedsize"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunkEmbedSize); + size_t MaxChunksPerBlock = Params["maxchunksperblock"sv].AsUInt64(RemoteStoreOptions::DefaultMaxChunksPerBlock); + size_t ChunkFileSizeLimit = Params["chunkfilesizelimit"sv].AsUInt64(RemoteStoreOptions::DefaultChunkFileSizeLimit); + bool Force = Params["force"sv].AsBool(false); + bool IgnoreMissingAttachments = Params["ignoremissingattachments"sv].AsBool(false); + bool EmbedLooseFile = Params["embedloosefiles"sv].AsBool(false); + + CreateRemoteStoreResult RemoteStoreResult = + CreateRemoteStore(Params, m_AuthMgr, MaxBlockSize, MaxChunkEmbedSize, Oplog->TempPath()); + + if (RemoteStoreResult.Store == nullptr) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + } + std::shared_ptr<RemoteProjectStore> RemoteStore = std::move(RemoteStoreResult.Store); + RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); + + JobId JobId = m_JobQueue.QueueJob( + fmt::format("Export oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), + [this, + ActualRemoteStore = std::move(RemoteStore), + Project, + Oplog, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments](JobContext& Context) { + Context.ReportMessage(fmt::format("Saving oplog '{}/{}' to {}, maxblocksize {}, maxchunkembedsize {}", + Project->Identifier, + Oplog->OplogId(), + ActualRemoteStore->GetInfo().Description, + NiceBytes(MaxBlockSize), + NiceBytes(MaxChunkEmbedSize))); + + WorkerThreadPool& WorkerPool = GetLargeWorkerPool(EWorkloadType::Background); + WorkerThreadPool& NetworkWorkerPool = GetMediumWorkerPool(EWorkloadType::Background); + + RemoteProjectStore::Result Result = SaveOplog(m_CidStore, + *ActualRemoteStore, + *Project, + *Oplog, + NetworkWorkerPool, + WorkerPool, + MaxBlockSize, + MaxChunksPerBlock, + MaxChunkEmbedSize, + ChunkFileSizeLimit, + EmbedLooseFile, + Force, + IgnoreMissingAttachments, + &Context); + auto Response = ConvertResult(Result); + ZEN_INFO("SaveOplog: Status: {} '{}'", ToString(Response.first), Response.second); + if (!IsHttpSuccessCode(Response.first)) + { + throw JobError(Response.second.empty() ? fmt::format("Status: {}", ToString(Response.first)) : Response.second, + (int)Response.first); + } + }); + + return HttpReq.WriteResponse(HttpResponseCode::Accepted, HttpContentType::kText, fmt::format("{}", JobId.Id)); + } + case HashStringDjb2("getchunks"sv): + { + RpcAcceptOptions AcceptFlags = static_cast<RpcAcceptOptions>(Cb["AcceptFlags"sv].AsUInt16(0u)); + int32_t TargetProcessId = Cb["Pid"sv].AsInt32(0); + + std::vector<ProjectStore::ChunkRequest> Requests = m_ProjectStore->ParseChunksRequests(*Project, *Oplog, Cb); + std::vector<ProjectStore::ChunkResult> Results = + Requests.empty() ? std::vector<ProjectStore::ChunkResult>{} : m_ProjectStore->GetChunks(*Project, *Oplog, Requests); + CbPackage Response = m_ProjectStore->WriteChunksRequestResponse(*Project, *Oplog, std::move(Requests), std::move(Results)); + + void* TargetProcessHandle = nullptr; + FormatFlags Flags = FormatFlags::kDefault; + if (EnumHasAllFlags(AcceptFlags, RpcAcceptOptions::kAllowLocalReferences)) + { + Flags |= FormatFlags::kAllowLocalReferences; + if (!EnumHasAnyFlags(AcceptFlags, RpcAcceptOptions::kAllowPartialLocalReferences)) + { + Flags |= FormatFlags::kDenyPartialLocalReferences; + } + TargetProcessHandle = m_OpenProcessCache.GetProcessHandle(HttpReq.SessionId(), TargetProcessId); + } + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(Response, Flags, TargetProcessHandle); + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); + } + case HashStringDjb2("putchunks"sv): + { + ZEN_TRACE_CPU("Store::Rpc::putchunks"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbObject Object = Package.GetObject(); + const bool UsingTempFiles = Object["usingtmpfiles"].AsBool(false); + + std::span<const CbAttachment> Attachments = Package.GetAttachments(); + if (!Attachments.empty()) + { + std::vector<IoBuffer> WriteAttachmentBuffers; + std::vector<IoHash> WriteRawHashes; + + WriteAttachmentBuffers.reserve(Attachments.size()); + WriteRawHashes.reserve(Attachments.size()); + + for (const CbAttachment& Attachment : Attachments) + { + IoHash RawHash = Attachment.GetHash(); + const CompressedBuffer& Compressed = Attachment.AsCompressedBinary(); + IoBuffer AttachmentPayload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + if (UsingTempFiles) + { + AttachmentPayload.SetDeleteOnClose(true); + } + WriteAttachmentBuffers.push_back(std::move(AttachmentPayload)); + WriteRawHashes.push_back(RawHash); + } + + Oplog->CaptureAddedAttachments(WriteRawHashes); + m_CidStore.AddChunks(WriteAttachmentBuffers, + WriteRawHashes, + UsingTempFiles ? CidStore::InsertMode::kMayBeMovedInPlace : CidStore::InsertMode::kCopyOnly); + } + return HttpReq.WriteResponse(HttpResponseCode::OK); + } + case HashStringDjb2("snapshot"sv): + { + ZEN_TRACE_CPU("Store::Rpc::snapshot"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + // Snapshot all referenced files. This brings the content of all + // files into the CID store + + uint32_t OpCount = 0; + uint64_t InlinedBytes = 0; + uint64_t InlinedFiles = 0; + uint64_t TotalBytes = 0; + uint64_t TotalFiles = 0; + + std::vector<CbObject> NewOps; + struct AddedChunk + { + IoBuffer Buffer; + uint64_t RawSize = 0; + }; + tsl::robin_map<IoHash, AddedChunk, IoHash::Hasher> AddedChunks; + + Oplog->IterateOplog( + [&](CbObjectView Op) { + bool OpRewritten = false; + bool AllOk = true; + + CbWriter FilesArrayWriter; + FilesArrayWriter.BeginArray("files"sv); + + for (CbFieldView& Field : Op["files"sv]) + { + bool CopyField = true; + + if (CbObjectView View = Field.AsObjectView()) + { + const IoHash DataHash = View["data"sv].AsHash(); + + if (DataHash == IoHash::Zero) + { + std::string_view ServerPath = View["serverpath"sv].AsString(); + std::filesystem::path FilePath = Project->RootDir / ServerPath; + BasicFile DataFile; + std::error_code Ec; + DataFile.Open(FilePath, BasicFile::Mode::kRead, Ec); + + if (Ec) + { + // Error... + + ZEN_ERROR("unable to read data from file '{}': {}", FilePath, Ec.message()); + + AllOk = false; + } + else + { + // Read file contents into memory, compress and keep in map of chunks to add to Cid store + IoBuffer FileIoBuffer = DataFile.ReadAll(); + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(std::move(FileIoBuffer))); + const uint64_t RawSize = Compressed.DecodeRawSize(); + const IoHash RawHash = Compressed.DecodeRawHash(); + if (!AddedChunks.contains(RawHash)) + { + const std::filesystem::path TempChunkPath = Oplog->TempPath() / RawHash.ToHexString(); + BasicFile ChunkTempFile; + ChunkTempFile.Open(TempChunkPath, BasicFile::Mode::kTruncateDelete); + ChunkTempFile.Write(Compressed.GetCompressed(), 0, Ec); + if (Ec) + { + Oid ChunkId = View["id"sv].AsObjectId(); + ZEN_ERROR("unable to write external file as compressed chunk '{}', id {}: {}", + FilePath, + ChunkId, + Ec.message()); + AllOk = false; + } + else + { + void* FileHandle = ChunkTempFile.Detach(); + IoBuffer ChunkBuffer(IoBuffer::File, + FileHandle, + 0, + Compressed.GetCompressed().GetSize(), + /*IsWholeFile*/ true); + ChunkBuffer.SetDeleteOnClose(true); + AddedChunks.insert_or_assign( + RawHash, + AddedChunk{.Buffer = std::move(ChunkBuffer), .RawSize = RawSize}); + } + } + + TotalBytes += RawSize; + ++TotalFiles; + + // Rewrite file array entry with new data reference + CbObjectWriter Writer(View.GetSize()); + RewriteCbObject(Writer, View, [&](CbObjectWriter&, CbFieldView Field) -> bool { + if (Field.GetName() == "data"sv) + { + // omit this field as we will write it explicitly ourselves + return true; + } + return false; + }); + Writer.AddBinaryAttachment("data"sv, RawHash); + + CbObject RewrittenOp = Writer.Save(); + FilesArrayWriter.AddObject(std::move(RewrittenOp)); + CopyField = false; + } + } + } + + if (CopyField) + { + FilesArrayWriter.AddField(Field); + } + else + { + OpRewritten = true; + } + } + + if (OpRewritten && AllOk) + { + FilesArrayWriter.EndArray(); + CbArray FilesArray = FilesArrayWriter.Save().AsArray(); + + CbObject RewrittenOp = RewriteCbObject(Op, [&](CbObjectWriter& NewWriter, CbFieldView Field) -> bool { + if (Field.GetName() == "files"sv) + { + NewWriter.AddArray("files"sv, FilesArray); + + return true; + } + + return false; + }); + + NewOps.push_back(std::move(RewrittenOp)); + } + + OpCount++; + }, + ProjectStore::Oplog::Paging{}); + + CbObjectWriter ResponseObj; + + // Persist rewritten oplog entries + if (!NewOps.empty()) + { + ResponseObj.BeginArray("rewritten_ops"); + + for (CbObject& NewOp : NewOps) + { + ProjectStore::LogSequenceNumber NewLsn = Oplog->AppendNewOplogEntry(std::move(NewOp)); + + ZEN_DEBUG("appended rewritten op at LSN: {}", NewLsn.Number); + + ResponseObj.AddInteger(NewLsn.Number); + } + + ResponseObj.EndArray(); + } + + // Ops that have moved chunks to a compressed buffer for storage in m_CidStore have been rewritten with references to the + // new chunk(s). Make sure we add the chunks to m_CidStore, and do it after we update the oplog so GC doesn't think we have + // unreferenced chunks. + for (auto It : AddedChunks) + { + const IoHash& RawHash = It.first; + AddedChunk& Chunk = It.second; + CidStore::InsertResult Result = m_CidStore.AddChunk(Chunk.Buffer, RawHash); + if (Result.New) + { + InlinedBytes += Chunk.RawSize; + ++InlinedFiles; + } + } + + ResponseObj << "inlined_bytes" << InlinedBytes << "inlined_files" << InlinedFiles; + ResponseObj << "total_bytes" << TotalBytes << "total_files" << TotalFiles; + + ZEN_INFO("oplog '{}/{}': rewrote {} oplog entries (out of {})", ProjectId, OplogId, NewOps.size(), OpCount); + + return HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); + } + case HashStringDjb2("appendops"sv): + { + ZEN_TRACE_CPU("Store::Rpc::appendops"); + if (!m_ProjectStore->AreDiskWritesAllowed()) + { + return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); + } + + CbArrayView OpsArray = Cb["ops"sv].AsArrayView(); + + size_t OpsBufferSize = 0; + for (CbFieldView OpView : OpsArray) + { + OpsBufferSize += OpView.GetSize(); + } + UniqueBuffer OpsBuffers = UniqueBuffer::Alloc(OpsBufferSize); + MutableMemoryView OpsBuffersMemory = OpsBuffers.GetMutableView(); + + std::vector<CbObjectView> Ops; + Ops.reserve(OpsArray.Num()); + for (CbFieldView OpView : OpsArray) + { + OpView.CopyTo(OpsBuffersMemory); + Ops.push_back(CbObjectView(OpsBuffersMemory.GetData())); + OpsBuffersMemory.MidInline(OpView.GetSize()); + } + + std::vector<ProjectStore::LogSequenceNumber> LSNs = Oplog->AppendNewOplogEntries(Ops); + ZEN_ASSERT(LSNs.size() == Ops.size()); + + std::vector<IoHash> MissingAttachments; + for (size_t OpIndex = 0; OpIndex < Ops.size(); OpIndex++) + { + if (LSNs[OpIndex]) + { + CbObjectView Op = Ops[OpIndex]; + Op.IterateAttachments([this, &MissingAttachments](CbFieldView AttachmentView) { + const IoHash Cid = AttachmentView.AsAttachment(); + if (!m_CidStore.ContainsChunk(Cid)) + { + MissingAttachments.push_back(Cid); + } + }); + } + } + + CbPackage ResponsePackage; + + { + CbObjectWriter ResponseObj; + ResponseObj.BeginArray("written_ops"); + + for (ProjectStore::LogSequenceNumber NewLsn : LSNs) + { + ZEN_DEBUG("appended written op at LSN: {}", NewLsn.Number); + ResponseObj.AddInteger(NewLsn.Number); + } + ResponseObj.EndArray(); + + if (!MissingAttachments.empty()) + { + ResponseObj.BeginArray("need"); + + for (const IoHash& Cid : MissingAttachments) + { + ResponseObj.AddHash(Cid); + } + ResponseObj.EndArray(); + } + ResponsePackage.SetObject(ResponseObj.Save()); + } + + std::vector<IoBuffer> ResponseBuffers = FormatPackageMessage(ResponsePackage); + + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, ResponseBuffers); + } + default: + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Unknown rpc method '{}'", Method)); + } +} +void +HttpProjectService::HandleDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::Details"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + Project.IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { + Oplog.IterateOplogWithKey( + [this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + }); + }); + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("projects"); + { + m_ProjectStore->DiscoverProjects(); + + m_ProjectStore->IterateProjects([&](ProjectStore::Project& Project) { + std::vector<std::string> OpLogs = Project.ScanForOplogs(); + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + }); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleProjectDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::ProjectDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + ProjectStore::Project& Project = *FoundProject.Get(); + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + FoundProject->IterateOplogs([&](const RwLock::SharedLockScope&, ProjectStore::Oplog& Oplog) { + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + }); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + std::vector<std::string> OpLogs = FoundProject->ScanForOplogs(); + Cbo.BeginArray("projects"); + { + CbWriteProject(m_CidStore, Project, OpLogs, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleOplogDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + Ref<ProjectStore::Oplog> FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + Oplog.IterateOplogWithKey([this, &Project, &Oplog, &CSVWriter, Details, AttachmentDetails](ProjectStore::LogSequenceNumber LSN, + const Oid& Key, + CbObjectView Op) { + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, Key, Op, CSVWriter); + }); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("oplogs"); + { + CbWriteOplog(m_CidStore, Oplog, Details, OpDetails, AttachmentDetails, Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +void +HttpProjectService::HandleOplogOpDetailsRequest(HttpRouterRequest& Req) +{ + ZEN_TRACE_CPU("ProjectService::OplogOpDetails"); + + using namespace std::literals; + + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto& ProjectId = Req.GetCapture(1); + const auto& OplogId = Req.GetCapture(2); + const auto& OpId = Req.GetCapture(3); + + HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); + bool CSV = Params.GetValue("csv"sv) == "true"sv; + bool Details = Params.GetValue("details"sv) == "true"sv; + bool OpDetails = Params.GetValue("opdetails"sv) == "true"sv; + bool AttachmentDetails = Params.GetValue("attachmentdetails"sv) == "true"sv; + + Ref<ProjectStore::Project> FoundProject = m_ProjectStore->OpenProject(ProjectId); + if (!FoundProject) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + Ref<ProjectStore::Oplog> FoundLog = FoundProject->OpenOplog(OplogId, /*AllowCompact*/ false, /*VerifyPathOnDisk*/ true); + if (!FoundLog) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + if (OpId.size() != 2 * sizeof(Oid::OidBits)) + { + m_ProjectStats.BadRequestCount++; + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Chunk info request for invalid chunk id '{}/{}'/'{}'", ProjectId, OplogId, OpId)); + } + + const Oid ObjId = Oid::FromHexString(OpId); + ProjectStore::Project& Project = *FoundProject.Get(); + ProjectStore::Oplog& Oplog = *FoundLog; + + std::optional<CbObject> Op = Oplog.GetOpByKey(ObjId); + if (!Op.has_value()) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + ProjectStore::LogSequenceNumber LSN = Oplog.GetOpIndexByKey(ObjId); + if (!LSN) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + if (CSV) + { + ExtendableStringBuilder<4096> CSVWriter; + CSVHeader(Details, AttachmentDetails, CSVWriter); + + CSVWriteOp(m_CidStore, Project.Identifier, Oplog.OplogId(), Details, AttachmentDetails, LSN, ObjId, Op.value(), CSVWriter); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, CSVWriter.ToView()); + } + else + { + CbObjectWriter Cbo; + Cbo.BeginArray("ops"); + { + CbWriteOp(m_CidStore, Details, OpDetails, AttachmentDetails, LSN, ObjId, Op.value(), Cbo); + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } +} + +} // namespace zen diff --git a/src/zenserver/storage/projectstore/httpprojectstore.h b/src/zenserver/storage/projectstore/httpprojectstore.h new file mode 100644 index 000000000..f0a0bcfa1 --- /dev/null +++ b/src/zenserver/storage/projectstore/httpprojectstore.h @@ -0,0 +1,111 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> +#include <zenstore/cidstore.h> + +namespace zen { + +class AuthMgr; +class JobQueue; +class OpenProcessCache; +class ProjectStore; + +////////////////////////////////////////////////////////////////////////// +// +// {project} a project identifier +// {target} a variation of the project, typically a build target +// {lsn} oplog entry sequence number +// +// /prj/{project} +// /prj/{project}/oplog/{target} +// /prj/{project}/oplog/{target}/{lsn} +// +// oplog entry +// +// id: {id} +// key: {} +// meta: {} +// data: [] +// refs: +// + +class HttpProjectService : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpProjectService(CidStore& Store, + ProjectStore* InProjectStore, + HttpStatusService& StatusService, + HttpStatsService& StatsService, + AuthMgr& AuthMgr, + OpenProcessCache& InOpenProcessCache, + JobQueue& InJobQueue); + ~HttpProjectService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct ProjectStats + { + std::atomic_uint64_t ProjectReadCount{}; + std::atomic_uint64_t ProjectWriteCount{}; + std::atomic_uint64_t ProjectDeleteCount{}; + std::atomic_uint64_t OpLogReadCount{}; + std::atomic_uint64_t OpLogWriteCount{}; + std::atomic_uint64_t OpLogDeleteCount{}; + std::atomic_uint64_t OpHitCount{}; + std::atomic_uint64_t OpMissCount{}; + std::atomic_uint64_t OpWriteCount{}; + std::atomic_uint64_t ChunkHitCount{}; + std::atomic_uint64_t ChunkMissCount{}; + std::atomic_uint64_t ChunkWriteCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + void HandleProjectListRequest(HttpRouterRequest& Req); + void HandleChunkBatchRequest(HttpRouterRequest& Req); + void HandleFilesRequest(HttpRouterRequest& Req); + void HandleChunkInfosRequest(HttpRouterRequest& Req); + void HandleChunkInfoRequest(HttpRouterRequest& Req); + void HandleChunkByIdRequest(HttpRouterRequest& Req); + void HandleChunkByCidRequest(HttpRouterRequest& Req); + void HandleOplogOpPrepRequest(HttpRouterRequest& Req); + void HandleOplogOpNewRequest(HttpRouterRequest& Req); + void HandleOplogValidateRequest(HttpRouterRequest& Req); + void HandleOpLogOpRequest(HttpRouterRequest& Req); + void HandleOpLogRequest(HttpRouterRequest& Req); + void HandleOpLogEntriesRequest(HttpRouterRequest& Req); + void HandleProjectRequest(HttpRouterRequest& Req); + void HandleOplogSaveRequest(HttpRouterRequest& Req); + void HandleOplogLoadRequest(HttpRouterRequest& Req); + void HandleRpcRequest(HttpRouterRequest& Req); + void HandleDetailsRequest(HttpRouterRequest& Req); + void HandleProjectDetailsRequest(HttpRouterRequest& Req); + void HandleOplogDetailsRequest(HttpRouterRequest& Req); + void HandleOplogOpDetailsRequest(HttpRouterRequest& Req); + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + CidStore& m_CidStore; + HttpRequestRouter m_Router; + Ref<ProjectStore> m_ProjectStore; + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + AuthMgr& m_AuthMgr; + OpenProcessCache& m_OpenProcessCache; + JobQueue& m_JobQueue; + ProjectStats m_ProjectStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp new file mode 100644 index 000000000..86bb09c21 --- /dev/null +++ b/src/zenserver/storage/storageconfig.cpp @@ -0,0 +1,1055 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "storageconfig.h" + +#include <zencore/basicfile.h> +#include <zencore/crypto.h> +#include <zencore/except.h> +#include <zencore/logging.h> + +#include "config/luaconfig.h" + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +#include <cxxopts.hpp> +#include <json11.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ValidateOptions(ZenStorageServerOptions& ServerOptions) +{ + if (ServerOptions.EncryptionKey.empty() == false) + { + const auto Key = AesKey256Bit::FromString(ServerOptions.EncryptionKey); + + if (Key.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-key' ('{}') is malformed", ServerOptions.EncryptionKey), {}); + } + } + + if (ServerOptions.EncryptionIV.empty() == false) + { + const auto IV = AesIV128Bit::FromString(ServerOptions.EncryptionIV); + + if (IV.IsValid() == false) + { + throw OptionParseException(fmt::format("'--encryption-aes-iv' ('{}') is malformed", ServerOptions.EncryptionIV), {}); + } + } + if (ServerOptions.HttpServerConfig.ForceLoopback && ServerOptions.IsDedicated) + { + throw OptionParseException("'--dedicated' conflicts with '--http-forceloopback'", {}); + } + if (ServerOptions.GcConfig.AttachmentPassCount > ZenGcConfig::GcMaxAttachmentPassCount) + { + throw OptionParseException(fmt::format("'--gc-attachment-passes' ('{}') is invalid, maximum is {}.", + ServerOptions.GcConfig.AttachmentPassCount, + ZenGcConfig::GcMaxAttachmentPassCount), + {}); + } + if (ServerOptions.GcConfig.UseGCV2 == false) + { + ZEN_WARN("'--gc-v2=false' is deprecated, reverting to '--gc-v2=true'"); + ServerOptions.GcConfig.UseGCV2 = true; + } +} + +class ZenStructuredCacheBucketsConfigOption : public LuaConfig::OptionValue +{ +public: + ZenStructuredCacheBucketsConfigOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const std::pair<std::string, ZenStructuredCacheBucketConfig>& Bucket : Value) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Bucket.first); + const ZenStructuredCacheBucketConfig& BucketConfig = Bucket.second; + + Writer.WriteValue("maxblocksize", fmt::format("{}", BucketConfig.MaxBlockSize)); + Writer.BeginContainer("memlayer"); + { + Writer.WriteValue("sizethreshold", fmt::format("{}", BucketConfig.MemCacheSizeThreshold)); + } + Writer.EndContainer(); + + Writer.WriteValue("payloadalignment", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("largeobjectthreshold", fmt::format("{}", BucketConfig.PayloadAlignment)); + Writer.WriteValue("limitoverwrites", fmt::format("{}", BucketConfig.LimitOverwrites)); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) + { + ZenStructuredCacheBucketConfig BucketConfig; + std::string Name = Kv.first.as<std::string>(); + if (Name.empty()) + { + throw OptionParseException("Cache bucket option must have a name.", {}); + } + + const uint64_t MaxBlockSize = Bucket.value().get_or("maxblocksize", BucketConfig.MaxBlockSize); + if (MaxBlockSize == 0) + { + throw OptionParseException( + fmt::format("'maxblocksize' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MaxBlockSize = MaxBlockSize; + + if (sol::optional<sol::table> Memlayer = Bucket.value().get_or("memlayer", sol::table())) + { + const uint64_t MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + if (MemCacheSizeThreshold == 0) + { + throw OptionParseException( + fmt::format("'memlayer.sizethreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.MemCacheSizeThreshold = Bucket.value().get_or("sizethreshold", BucketConfig.MemCacheSizeThreshold); + } + + const uint32_t PayloadAlignment = Bucket.value().get_or("payloadalignment", BucketConfig.PayloadAlignment); + if (PayloadAlignment == 0 || !IsPow2(PayloadAlignment)) + { + throw OptionParseException( + fmt::format( + "'payloadalignment' option for cache bucket '{}' is invalid. It needs to be non-zero and a power of two.", + Name), + {}); + } + BucketConfig.PayloadAlignment = PayloadAlignment; + + const uint64_t LargeObjectThreshold = Bucket.value().get_or("largeobjectthreshold", BucketConfig.LargeObjectThreshold); + if (LargeObjectThreshold == 0) + { + throw OptionParseException( + fmt::format("'largeobjectthreshold' option for cache bucket '{}' is invalid. It must be non-zero.", Name), + {}); + } + BucketConfig.LargeObjectThreshold = LargeObjectThreshold; + + BucketConfig.LimitOverwrites = Bucket.value().get_or("limitoverwrites", BucketConfig.LimitOverwrites); + + Value.push_back(std::make_pair(std::move(Name), BucketConfig)); + } + } + } + } + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value; +}; + +UpstreamCachePolicy +ParseUpstreamCachePolicy(std::string_view Options) +{ + if (Options == "readonly") + { + return UpstreamCachePolicy::Read; + } + else if (Options == "writeonly") + { + return UpstreamCachePolicy::Write; + } + else if (Options == "disabled") + { + return UpstreamCachePolicy::Disabled; + } + else + { + return UpstreamCachePolicy::ReadWrite; + } +} + +ZenObjectStoreConfig +ParseBucketConfigs(std::span<std::string> Buckets) +{ + using namespace std::literals; + + ZenObjectStoreConfig Cfg; + + // split bucket args in the form of "{BucketName};{LocalPath}" + for (std::string_view Bucket : Buckets) + { + ZenObjectStoreConfig::BucketConfig NewBucket; + + if (auto Idx = Bucket.find_first_of(";"); Idx != std::string_view::npos) + { + NewBucket.Name = Bucket.substr(0, Idx); + NewBucket.Directory = Bucket.substr(Idx + 1); + } + else + { + NewBucket.Name = Bucket; + } + + Cfg.Buckets.push_back(std::move(NewBucket)); + } + + return Cfg; +} + +class CachePolicyOption : public LuaConfig::OptionValue +{ +public: + CachePolicyOption(UpstreamCachePolicy& Value) : Value(Value) {} + virtual void Print(std::string_view, StringBuilderBase& StringBuilder) override + { + switch (Value) + { + case UpstreamCachePolicy::Read: + StringBuilder.Append("readonly"); + break; + case UpstreamCachePolicy::Write: + StringBuilder.Append("writeonly"); + break; + case UpstreamCachePolicy::Disabled: + StringBuilder.Append("disabled"); + break; + case UpstreamCachePolicy::ReadWrite: + StringBuilder.Append("readwrite"); + break; + default: + ZEN_ASSERT(false); + } + } + virtual void Parse(sol::object Object) override + { + std::string PolicyString = Object.as<std::string>(); + if (PolicyString == "readonly") + { + Value = UpstreamCachePolicy::Read; + } + else if (PolicyString == "writeonly") + { + Value = UpstreamCachePolicy::Write; + } + else if (PolicyString == "disabled") + { + Value = UpstreamCachePolicy::Disabled; + } + else if (PolicyString == "readwrite") + { + Value = UpstreamCachePolicy::ReadWrite; + } + } + UpstreamCachePolicy& Value; +}; + +class ZenAuthConfigOption : public LuaConfig::OptionValue +{ +public: + ZenAuthConfigOption(ZenAuthConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.OpenIdProviders.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenOpenIdProviderConfig& Config : Value.OpenIdProviders) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + Writer.WriteValue("url", Config.Url); + Writer.WriteValue("clientid", Config.ClientId); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> OpenIdProviders = Object.as<sol::table>()) + { + for (const auto& Kv : OpenIdProviders.value()) + { + if (sol::optional<sol::table> OpenIdProvider = Kv.second.as<sol::table>()) + { + std::string Name = OpenIdProvider.value().get_or("name", std::string("Default")); + std::string Url = OpenIdProvider.value().get_or("url", std::string()); + std::string ClientId = OpenIdProvider.value().get_or("clientid", std::string()); + + Value.OpenIdProviders.push_back({.Name = std::move(Name), .Url = std::move(Url), .ClientId = std::move(ClientId)}); + } + } + } + } + ZenAuthConfig& Value; +}; + +class ZenObjectStoreConfigOption : public LuaConfig::OptionValue +{ +public: + ZenObjectStoreConfigOption(ZenObjectStoreConfig& Value) : Value(Value) {} + virtual void Print(std::string_view Indent, StringBuilderBase& StringBuilder) override + { + if (Value.Buckets.empty()) + { + StringBuilder.Append("{}"); + return; + } + LuaConfig::LuaContainerWriter Writer(StringBuilder, Indent); + for (const ZenObjectStoreConfig::BucketConfig& Config : Value.Buckets) + { + Writer.BeginContainer(""); + { + Writer.WriteValue("name", Config.Name); + std::string Directory = Config.Directory.string(); + LuaConfig::EscapeBackslash(Directory); + Writer.WriteValue("directory", Directory); + } + Writer.EndContainer(); + } + } + virtual void Parse(sol::object Object) override + { + if (sol::optional<sol::table> Buckets = Object.as<sol::table>()) + { + for (const auto& Kv : Buckets.value()) + { + if (sol::optional<sol::table> Bucket = Kv.second.as<sol::table>()) + { + std::string Name = Bucket.value().get_or("name", std::string("Default")); + std::string Directory = Bucket.value().get_or("directory", std::string()); + + Value.Buckets.push_back({.Name = std::move(Name), .Directory = MakeSafeAbsolutePath(Directory)}); + } + } + } + } + ZenObjectStoreConfig& Value; +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(UpstreamCachePolicy& Value) +{ + return std::make_shared<CachePolicyOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(ZenAuthConfig& Value) +{ + return std::make_shared<ZenAuthConfigOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(ZenObjectStoreConfig& Value) +{ + return std::make_shared<ZenObjectStoreConfigOption>(Value); +}; + +std::shared_ptr<LuaConfig::OptionValue> +MakeOption(std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>>& Value) +{ + return std::make_shared<ZenStructuredCacheBucketsConfigOption>(Value); +}; + +void +ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile) +{ + ZEN_TRACE_CPU("ParseConfigFile"); + + using namespace std::literals; + + LuaConfig::Options LuaOptions; + + AddServerConfigOptions(LuaOptions, ServerOptions); + + ////// server + LuaOptions.AddOption("server.pluginsconfigfile"sv, ServerOptions.PluginsConfigFile, "plugins-config"sv); + + ////// objectstore + LuaOptions.AddOption("server.objectstore.enabled"sv, ServerOptions.ObjectStoreEnabled, "objectstore-enabled"sv); + LuaOptions.AddOption("server.objectstore.buckets"sv, ServerOptions.ObjectStoreConfig); + + ////// buildsstore + LuaOptions.AddOption("server.buildstore.enabled"sv, ServerOptions.BuildStoreConfig.Enabled, "buildstore-enabled"sv); + LuaOptions.AddOption("server.buildstore.disksizelimit"sv, ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit, "buildstore-disksizelimit"); + + ////// cache + LuaOptions.AddOption("cache.enable"sv, ServerOptions.StructuredCacheConfig.Enabled); + LuaOptions.AddOption("cache.writelog"sv, ServerOptions.StructuredCacheConfig.WriteLogEnabled, "cache-write-log"sv); + LuaOptions.AddOption("cache.accesslog"sv, ServerOptions.StructuredCacheConfig.AccessLogEnabled, "cache-access-log"sv); + + LuaOptions.AddOption("cache.buckets"sv, ServerOptions.StructuredCacheConfig.PerBucketConfigs, "cache.buckets"sv); + + LuaOptions.AddOption("cache.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.memlayer.targetfootprint"sv, + ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes, + "cache-memlayer-targetfootprint"sv); + LuaOptions.AddOption("cache.memlayer.triminterval"sv, + ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds, + "cache-memlayer-triminterval"sv); + LuaOptions.AddOption("cache.memlayer.maxage"sv, ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds, "cache-memlayer-maxage"sv); + + LuaOptions.AddOption("cache.bucket.maxblocksize"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + "cache-bucket-maxblocksize"sv); + LuaOptions.AddOption("cache.bucket.memlayer.sizethreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + "cache-bucket-memlayer-sizethreshold"sv); + LuaOptions.AddOption("cache.bucket.payloadalignment"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + "cache-bucket-payloadalignment"sv); + LuaOptions.AddOption("cache.bucket.largeobjectthreshold"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + "cache-bucket-largeobjectthreshold"sv); + LuaOptions.AddOption("cache.bucket.limitoverwrites"sv, + ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites, + "cache-bucket-limit-overwrites"sv); + + ////// cache.upstream + LuaOptions.AddOption("cache.upstream.policy"sv, ServerOptions.UpstreamCacheConfig.CachePolicy, "upstream-cache-policy"sv); + LuaOptions.AddOption("cache.upstream.upstreamthreadcount"sv, + ServerOptions.UpstreamCacheConfig.UpstreamThreadCount, + "upstream-thread-count"sv); + LuaOptions.AddOption("cache.upstream.connecttimeoutms"sv, + ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds, + "upstream-connect-timeout-ms"sv); + LuaOptions.AddOption("cache.upstream.timeoutms"sv, ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds, "upstream-timeout-ms"sv); + + ////// cache.upstream.jupiter + LuaOptions.AddOption("cache.upstream.jupiter.name"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Name); + LuaOptions.AddOption("cache.upstream.jupiter.url"sv, ServerOptions.UpstreamCacheConfig.JupiterConfig.Url, "upstream-jupiter-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl, + "upstream-jupiter-oauth-url"sv); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientid"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId, + "upstream-jupiter-oauth-clientid"); + LuaOptions.AddOption("cache.upstream.jupiter.oauthclientsecret"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret, + "upstream-jupiter-oauth-clientsecret"sv); + LuaOptions.AddOption("cache.upstream.jupiter.openidprovider"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider, + "upstream-jupiter-openid-provider"sv); + LuaOptions.AddOption("cache.upstream.jupiter.token"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken, + "upstream-jupiter-token"sv); + LuaOptions.AddOption("cache.upstream.jupiter.namespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace, + "upstream-jupiter-namespace"sv); + LuaOptions.AddOption("cache.upstream.jupiter.ddcnamespace"sv, + ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace, + "upstream-jupiter-namespace-ddc"sv); + + ////// cache.upstream.zen + // LuaOptions.AddOption("cache.upstream.zen"sv, ServerOptions.UpstreamCacheConfig.ZenConfig); + LuaOptions.AddOption("cache.upstream.zen.name"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Name); + LuaOptions.AddOption("cache.upstream.zen.dns"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Dns); + LuaOptions.AddOption("cache.upstream.zen.url"sv, ServerOptions.UpstreamCacheConfig.ZenConfig.Urls); + + ////// gc + LuaOptions.AddOption("gc.enabled"sv, ServerOptions.GcConfig.Enabled, "gc-enabled"sv); + LuaOptions.AddOption("gc.v2"sv, ServerOptions.GcConfig.UseGCV2, "gc-v2"sv); + + LuaOptions.AddOption("gc.monitorintervalseconds"sv, ServerOptions.GcConfig.MonitorIntervalSeconds, "gc-monitor-interval-seconds"sv); + LuaOptions.AddOption("gc.intervalseconds"sv, ServerOptions.GcConfig.IntervalSeconds, "gc-interval-seconds"sv); + LuaOptions.AddOption("gc.collectsmallobjects"sv, ServerOptions.GcConfig.CollectSmallObjects, "gc-small-objects"sv); + LuaOptions.AddOption("gc.diskreservesize"sv, ServerOptions.GcConfig.DiskReserveSize, "disk-reserve-size"sv); + LuaOptions.AddOption("gc.disksizesoftlimit"sv, ServerOptions.GcConfig.DiskSizeSoftLimit, "gc-disksize-softlimit"sv); + LuaOptions.AddOption("gc.lowdiskspacethreshold"sv, + ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, + "gc-low-diskspace-threshold"sv); + LuaOptions.AddOption("gc.lightweightintervalseconds"sv, + ServerOptions.GcConfig.LightweightIntervalSeconds, + "gc-lightweight-interval-seconds"sv); + LuaOptions.AddOption("gc.compactblockthreshold"sv, + ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + "gc-compactblock-threshold"sv); + LuaOptions.AddOption("gc.verbose"sv, ServerOptions.GcConfig.Verbose, "gc-verbose"sv); + LuaOptions.AddOption("gc.single-threaded"sv, ServerOptions.GcConfig.SingleThreaded, "gc-single-threaded"sv); + LuaOptions.AddOption("gc.cache.attachment.store"sv, ServerOptions.GcConfig.StoreCacheAttachmentMetaData, "gc-cache-attachment-store"); + LuaOptions.AddOption("gc.projectstore.attachment.store"sv, + ServerOptions.GcConfig.StoreProjectAttachmentMetaData, + "gc-projectstore-attachment-store"); + LuaOptions.AddOption("gc.attachment.passes"sv, ServerOptions.GcConfig.AttachmentPassCount, "gc-attachment-passes"sv); + LuaOptions.AddOption("gc.validation"sv, ServerOptions.GcConfig.EnableValidation, "gc-validation"); + + LuaOptions.AddOption("gc.cache.maxdurationseconds"sv, ServerOptions.GcConfig.Cache.MaxDurationSeconds, "gc-cache-duration-seconds"sv); + LuaOptions.AddOption("gc.projectstore.duration.seconds"sv, + ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds, + "gc-projectstore-duration-seconds"); + LuaOptions.AddOption("gc.buildstore.duration.seconds"sv, + ServerOptions.GcConfig.BuildStore.MaxDurationSeconds, + "gc-buildstore-duration-seconds"); + + ////// security + LuaOptions.AddOption("security.encryptionaeskey"sv, ServerOptions.EncryptionKey, "encryption-aes-key"sv); + LuaOptions.AddOption("security.encryptionaesiv"sv, ServerOptions.EncryptionIV, "encryption-aes-iv"sv); + LuaOptions.AddOption("security.openidproviders"sv, ServerOptions.AuthConfig); + + ////// workspaces + LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); + LuaOptions.AddOption("workspaces.allowconfigchanges"sv, + ServerOptions.WorksSpacesConfig.AllowConfigurationChanges, + "workspaces-allow-changes"sv); + + LuaOptions.Parse(Path, CmdLineResult); + + // These have special command line processing so we make sure we export them if they were configured on command line + if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) + { + LuaOptions.Touch("security.openidproviders"sv); + } + if (!ServerOptions.ObjectStoreConfig.Buckets.empty()) + { + LuaOptions.Touch("server.objectstore.buckets"sv); + } + if (!ServerOptions.StructuredCacheConfig.PerBucketConfigs.empty()) + { + LuaOptions.Touch("cache.buckets"sv); + } + + if (!OutputConfigFile.empty()) + { + std::filesystem::path WritePath(MakeSafeAbsolutePath(OutputConfigFile)); + ExtendableStringBuilder<512> ConfigStringBuilder; + LuaOptions.Print(ConfigStringBuilder, CmdLineResult); + BasicFile Output; + Output.Open(WritePath, BasicFile::Mode::kTruncate); + Output.Write(ConfigStringBuilder.Data(), ConfigStringBuilder.Size(), 0); + } +} + +void +ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort) +{ + using namespace std::literals; + + IoBuffer Body = IoBufferBuilder::MakeFromFile(Path); + std::string JsonText(reinterpret_cast<const char*>(Body.GetData()), Body.GetSize()); + std::string JsonError; + json11::Json PluginsInfo = json11::Json::parse(JsonText, JsonError); + if (!JsonError.empty()) + { + ZEN_WARN("failed parsing plugins config file '{}'. Reason: '{}'", Path, JsonError); + return; + } + for (const json11::Json& PluginInfo : PluginsInfo.array_items()) + { + if (!PluginInfo.is_object()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugin definition, object expected, got '{}'", Path, PluginInfo.dump()); + continue; + } + + HttpServerPluginConfig Config = {}; + + bool bNeedsPort = true; + + for (const std::pair<const std::string, json11::Json>& Items : PluginInfo.object_items()) + { + if (!Items.second.is_string()) + { + ZEN_WARN("the json file '{}' does not contain a valid plugins definition, string expected, got '{}'", + Path, + Items.second.dump()); + continue; + } + + const std::string& Name = Items.first; + const std::string& Value = Items.second.string_value(); + + if (Name == "name"sv) + Config.PluginName = Value; + else + { + Config.PluginOptions.push_back({Name, Value}); + + if (Name == "port"sv) + { + bNeedsPort = false; + } + } + } + + // add a default base port in case if json config didn't provide one + if (bNeedsPort) + { + Config.PluginOptions.push_back({"port", std::to_string(BasePort)}); + } + + ServerOptions.HttpServerConfig.PluginConfigs.push_back(Config); + } +} + +void +ZenStorageServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_options()("snapshot-dir", + "Specify a snapshot of server state to mirror into the persistence root at startup", + cxxopts::value<std::string>(BaseSnapshotDir)); + options.add_options()("plugins-config", "Path to plugins config file", cxxopts::value<std::string>(PluginsConfigFile)); + options.add_options()("scrub", + "Validate state at startup", + cxxopts::value(ServerOptions.ScrubOptions)->implicit_value("yes"), + "(nocas,nogc,nodelete,yes,no)*"); + + AddSecurityOptions(options, ServerOptions); + AddCacheOptions(options, ServerOptions); + AddGcOptions(options, ServerOptions); + AddObjectStoreOptions(options, ServerOptions); + AddBuildStoreOptions(options, ServerOptions); + AddWorkspacesOptions(options, ServerOptions); +} + +void +ZenStorageServerCmdLineOptions::AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("security", + "", + "encryption-aes-key", + "256 bit AES encryption key", + cxxopts::value<std::string>(ServerOptions.EncryptionKey), + ""); + + options.add_option("security", + "", + "encryption-aes-iv", + "128 bit AES encryption initialization vector", + cxxopts::value<std::string>(ServerOptions.EncryptionIV), + ""); + + options.add_option("security", + "", + "openid-provider-name", + "Open ID provider name", + cxxopts::value<std::string>(OpenIdProviderName), + "Default"); + + options.add_option("security", "", "openid-provider-url", "Open ID provider URL", cxxopts::value<std::string>(OpenIdProviderUrl), ""); + options.add_option("security", "", "openid-client-id", "Open ID client ID", cxxopts::value<std::string>(OpenIdClientId), ""); +} + +void +ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("cache", + "", + "upstream-cache-policy", + "", + cxxopts::value<std::string>(UpstreamCachePolicyOptions)->default_value(""), + "Upstream cache policy (readwrite|readonly|writeonly|disabled)"); + + options.add_option("cache", + "", + "upstream-jupiter-url", + "URL to a Jupiter instance", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Url)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-url", + "URL to the OAuth provier", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthUrl)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientid", + "The OAuth client ID", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientId)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-oauth-clientsecret", + "The OAuth client secret", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OAuthClientSecret)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-openid-provider", + "Name of a registered Open ID provider", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.OpenIdProvider)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-token", + "A static authentication token", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.AccessToken)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace", + "The Common Blob Store API namespace", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.Namespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-jupiter-namespace-ddc", + "The lecacy DDC namespace", + cxxopts::value<std::string>(ServerOptions.UpstreamCacheConfig.JupiterConfig.DdcNamespace)->default_value(""), + ""); + + options.add_option("cache", + "", + "upstream-zen-url", + "URL to remote Zen server. Use a comma separated list to choose the one with the best latency.", + cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Urls), + ""); + + options.add_option("cache", + "", + "upstream-zen-dns", + "DNS that resolves to one or more Zen server instance(s)", + cxxopts::value<std::vector<std::string>>(ServerOptions.UpstreamCacheConfig.ZenConfig.Dns), + ""); + + options.add_option("cache", + "", + "upstream-thread-count", + "Number of threads used for upstream procsssing", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.UpstreamThreadCount)->default_value("4"), + ""); + + options.add_option("cache", + "", + "upstream-connect-timeout-ms", + "Connect timeout in millisecond(s). Default 5000 ms.", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.ConnectTimeoutMilliseconds)->default_value("5000"), + ""); + + options.add_option("cache", + "", + "upstream-timeout-ms", + "Timeout in millisecond(s). Default 0 ms", + cxxopts::value<int32_t>(ServerOptions.UpstreamCacheConfig.TimeoutMilliseconds)->default_value("0"), + ""); + + options.add_option("cache", + "", + "cache-write-log", + "Whether cache write log is enabled", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.WriteLogEnabled)->default_value("false"), + ""); + + options.add_option("cache", + "", + "cache-access-log", + "Whether cache access log is enabled", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.AccessLogEnabled)->default_value("false"), + ""); + + options.add_option( + "cache", + "", + "cache-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching. " + "Obsolete, replaced by `--cache-bucket-memlayer-sizethreshold`", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-targetfootprint", + "Max allowed memory used by cache memory layer per namespace in bytes. Default set to 536870912 (512 Mb).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes)->default_value("536870912"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-triminterval", + "Minimum time between each attempt to trim cache memory layers in seconds. Default set to 60 (1 min).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds)->default_value("60"), + ""); + + options.add_option("cache", + "", + "cache-memlayer-maxage", + "Maximum age of payloads when trimming cache memory layers in seconds. Default set to 86400 (1 day).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), + ""); + + options.add_option("cache", + "", + "cache-bucket-maxblocksize", + "Max size of cache bucket blocks. Default set to 1073741824 (1GB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize)->default_value("1073741824"), + ""); + + options.add_option("cache", + "", + "cache-bucket-payloadalignment", + "Payload alignement for cache bucket blocks. Default set to 16.", + cxxopts::value<uint32_t>(ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment)->default_value("16"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-largeobjectthreshold", + "Threshold for storing cache bucket values as loose files. Default set to 131072 (128 KB).", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold)->default_value("131072"), + ""); + + options.add_option( + "cache", + "", + "cache-bucket-memlayer-sizethreshold", + "The largest size of a cache entry that may be cached in memory. Default set to 1024 (1 Kb). Set to 0 to disable memory " + "caching.", + cxxopts::value<uint64_t>(ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold)->default_value("1024"), + ""); + + options.add_option("cache", + "", + "cache-bucket-limit-overwrites", + "Whether to require policy flag pattern before allowing overwrites in cache bucket", + cxxopts::value<bool>(ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites)->default_value("false"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("gc", + "", + "gc-cache-attachment-store", + "Enable storing attachments referenced by a cache record in block store meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreCacheAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-attachment-store", + "Enable storing attachments referenced by project oplogs in meta data.", + cxxopts::value<bool>(ServerOptions.GcConfig.StoreProjectAttachmentMetaData)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-validation", + "Enable validation of references after full GC.", + cxxopts::value<bool>(ServerOptions.GcConfig.EnableValidation)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-enabled", + "Whether garbage collection is enabled or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.Enabled)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-v2", + "Use V2 of GC implementation or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.UseGCV2)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-small-objects", + "Whether garbage collection of small objects is enabled or not.", + cxxopts::value<bool>(ServerOptions.GcConfig.CollectSmallObjects)->default_value("true"), + ""); + + options.add_option("gc", + "", + "gc-interval-seconds", + "Garbage collection interval in seconds. Default set to 3600 (1 hour).", + cxxopts::value<int32_t>(ServerOptions.GcConfig.IntervalSeconds)->default_value("3600"), + ""); + + options.add_option("gc", + "", + "gc-lightweight-interval-seconds", + "Lightweight garbage collection interval in seconds. Default set to 900 (30 min).", + cxxopts::value<int32_t>(ServerOptions.GcConfig.LightweightIntervalSeconds)->default_value("900"), + ""); + + options.add_option("gc", + "", + "gc-cache-duration-seconds", + "Max duration in seconds before Z$ entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.Cache.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-projectstore-duration-seconds", + "Max duration in seconds before project store entries get evicted. Default set to 1209600 (2 weeks)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds)->default_value("1209600"), + ""); + + options.add_option("gc", + "", + "gc-buildstore-duration-seconds", + "Max duration in seconds before build store entries get evicted. Default set to 604800 (1 week)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds)->default_value("604800"), + ""); + + options.add_option("gc", + "", + "disk-reserve-size", + "Size of gc disk reserve in bytes. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskReserveSize)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-monitor-interval-seconds", + "Garbage collection monitoring interval in seconds. Default set to 30 (30 seconds)", + cxxopts::value<int32_t>(ServerOptions.GcConfig.MonitorIntervalSeconds)->default_value("30"), + ""); + + options.add_option("gc", + "", + "gc-low-diskspace-threshold", + "Minimum free space on disk to allow writes to disk. Default set to 268435456 (256 Mb). Set to zero to disable.", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites)->default_value("268435456"), + ""); + + options.add_option("gc", + "", + "gc-disksize-softlimit", + "Garbage collection disk usage soft limit. Default set to 0 (Off).", + cxxopts::value<uint64_t>(ServerOptions.GcConfig.DiskSizeSoftLimit)->default_value("0"), + ""); + + options.add_option("gc", + "", + "gc-compactblock-threshold", + "Garbage collection - how much of a compact block should be used to skip compacting the block. 0 - compact only " + "empty eligible blocks, 100 - compact all non-full eligible blocks.", + cxxopts::value<uint32_t>(ServerOptions.GcConfig.CompactBlockUsageThresholdPercent)->default_value("60"), + ""); + + options.add_option("gc", + "", + "gc-verbose", + "Enable verbose logging for GC.", + cxxopts::value<bool>(ServerOptions.GcConfig.Verbose)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-single-threaded", + "Force GC to run single threaded.", + cxxopts::value<bool>(ServerOptions.GcConfig.SingleThreaded)->default_value("false"), + ""); + + options.add_option("gc", + "", + "gc-attachment-passes", + "Limit the range of unreferenced attachments included in GC check by breaking it into passes. Default is one pass " + "which includes all the attachments.", + cxxopts::value<uint16_t>(ServerOptions.GcConfig.AttachmentPassCount)->default_value("1"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("objectstore", + "", + "objectstore-enabled", + "Whether the object store is enabled or not.", + cxxopts::value<bool>(ServerOptions.ObjectStoreEnabled)->default_value("false"), + ""); + + options.add_option("objectstore", + "", + "objectstore-bucket", + "Object store bucket mappings.", + cxxopts::value<std::vector<std::string>>(BucketConfigs), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("buildstore", + "", + "buildstore-enabled", + "Whether the builds store is enabled or not.", + cxxopts::value<bool>(ServerOptions.BuildStoreConfig.Enabled)->default_value("false"), + ""); + options.add_option("buildstore", + "", + "buildstore-disksizelimit", + "Max number of bytes before build store entries get evicted. Default set to 1099511627776 (1TB week)", + cxxopts::value<uint64_t>(ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit)->default_value("1099511627776"), + ""); +} + +void +ZenStorageServerCmdLineOptions::AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + options.add_option("workspaces", + "", + "workspaces-enabled", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.Enabled)->default_value("true"), + "Enable workspaces support with folder sharing"); + + options.add_option("workspaces", + "", + "workspaces-allow-changes", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.AllowConfigurationChanges)->default_value("false"), + "Allow adding/modifying/deleting of workspace and shares via http endpoint"); +} + +void +ZenStorageServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions) +{ + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.PluginsConfigFile = MakeSafeAbsolutePath(PluginsConfigFile); + ServerOptions.UpstreamCacheConfig.CachePolicy = ParseUpstreamCachePolicy(UpstreamCachePolicyOptions); + + if (!BaseSnapshotDir.empty()) + { + if (ServerOptions.DataDir.empty()) + throw OptionParseException("'--snapshot-dir' requires '--data-dir'", options.help()); + + if (!IsDir(ServerOptions.BaseSnapshotDir)) + throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir)); + } + + if (OpenIdProviderUrl.empty() == false) + { + if (OpenIdClientId.empty()) + { + throw OptionParseException("'--openid-provider-url' requires '--openid-client-id'", options.help()); + } + + ServerOptions.AuthConfig.OpenIdProviders.push_back( + {.Name = OpenIdProviderName, .Url = OpenIdProviderUrl, .ClientId = OpenIdClientId}); + } + + ServerOptions.ObjectStoreConfig = ParseBucketConfigs(BucketConfigs); +} + +} // namespace zen diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h new file mode 100644 index 000000000..ca0cf4135 --- /dev/null +++ b/src/zenserver/storage/storageconfig.h @@ -0,0 +1,203 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "config/config.h" + +namespace zen { + +struct ZenUpstreamJupiterConfig +{ + std::string Name; + std::string Url; + std::string OAuthUrl; + std::string OAuthClientId; + std::string OAuthClientSecret; + std::string OpenIdProvider; + std::string AccessToken; + std::string Namespace; + std::string DdcNamespace; +}; + +struct ZenUpstreamZenConfig +{ + std::string Name; + std::vector<std::string> Urls; + std::vector<std::string> Dns; +}; + +enum class UpstreamCachePolicy : uint8_t +{ + Disabled = 0, + Read = 1 << 0, + Write = 1 << 1, + ReadWrite = Read | Write +}; + +struct ZenUpstreamCacheConfig +{ + ZenUpstreamJupiterConfig JupiterConfig; + ZenUpstreamZenConfig ZenConfig; + int32_t UpstreamThreadCount = 4; + int32_t ConnectTimeoutMilliseconds = 5000; + int32_t TimeoutMilliseconds = 0; + UpstreamCachePolicy CachePolicy = UpstreamCachePolicy::ReadWrite; +}; + +struct ZenCacheEvictionPolicy +{ + int32_t MaxDurationSeconds = 24 * 60 * 60; +}; + +struct ZenProjectStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 7 * 24 * 60 * 60; +}; + +struct ZenBuildStoreEvictionPolicy +{ + int32_t MaxDurationSeconds = 3 * 24 * 60 * 60; +}; + +struct ZenGcConfig +{ + // ZenCasEvictionPolicy Cas; + ZenCacheEvictionPolicy Cache; + ZenProjectStoreEvictionPolicy ProjectStore; + ZenBuildStoreEvictionPolicy BuildStore; + int32_t MonitorIntervalSeconds = 30; + int32_t IntervalSeconds = 0; + bool CollectSmallObjects = true; + bool Enabled = true; + uint64_t DiskReserveSize = 1ul << 28; + uint64_t DiskSizeSoftLimit = 0; + int32_t LightweightIntervalSeconds = 0; + uint64_t MinimumFreeDiskSpaceToAllowWrites = 1ul << 28; + bool UseGCV2 = false; + uint32_t CompactBlockUsageThresholdPercent = 90; + bool Verbose = false; + bool SingleThreaded = false; + static constexpr uint16_t GcMaxAttachmentPassCount = 256; + uint16_t AttachmentPassCount = 1; + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; + bool EnableValidation = true; +}; + +struct ZenOpenIdProviderConfig +{ + std::string Name; + std::string Url; + std::string ClientId; +}; + +struct ZenAuthConfig +{ + std::vector<ZenOpenIdProviderConfig> OpenIdProviders; +}; + +struct ZenObjectStoreConfig +{ + struct BucketConfig + { + std::string Name; + std::filesystem::path Directory; + }; + + std::vector<BucketConfig> Buckets; +}; + +struct ZenStructuredCacheBucketConfig +{ + uint64_t MaxBlockSize = 1ull << 30; + uint32_t PayloadAlignment = 1u << 4; + uint64_t MemCacheSizeThreshold = 1 * 1024; + uint64_t LargeObjectThreshold = 128 * 1024; + bool LimitOverwrites = false; +}; + +struct ZenStructuredCacheConfig +{ + bool Enabled = true; + bool WriteLogEnabled = false; + bool AccessLogEnabled = false; + std::vector<std::pair<std::string, ZenStructuredCacheBucketConfig>> PerBucketConfigs; + ZenStructuredCacheBucketConfig BucketConfig; + uint64_t MemTargetFootprintBytes = 512 * 1024 * 1024; + uint64_t MemTrimIntervalSeconds = 60; + uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); +}; + +struct ZenProjectStoreConfig +{ + bool StoreCacheAttachmentMetaData = false; + bool StoreProjectAttachmentMetaData = false; +}; + +struct ZenBuildStoreConfig +{ + bool Enabled = false; + uint64_t MaxDiskSpaceLimit = 1u * 1024u * 1024u * 1024u * 1024u; // 1TB +}; + +struct ZenWorkspacesConfig +{ + bool Enabled = false; + bool AllowConfigurationChanges = false; +}; + +struct ZenStorageServerOptions : public ZenServerOptions +{ + ZenUpstreamCacheConfig UpstreamCacheConfig; + ZenGcConfig GcConfig; + ZenAuthConfig AuthConfig; + ZenObjectStoreConfig ObjectStoreConfig; + ZenStructuredCacheConfig StructuredCacheConfig; + ZenProjectStoreConfig ProjectStoreConfig; + ZenBuildStoreConfig BuildStoreConfig; + ZenWorkspacesConfig WorksSpacesConfig; + std::filesystem::path PluginsConfigFile; // Path to plugins config file + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + bool ObjectStoreEnabled = false; + std::string ScrubOptions; +}; + +void ParseConfigFile(const std::filesystem::path& Path, + ZenStorageServerOptions& ServerOptions, + const cxxopts::ParseResult& CmdLineResult, + std::string_view OutputConfigFile); + +void ParsePluginsConfigFile(const std::filesystem::path& Path, ZenStorageServerOptions& ServerOptions, int BasePort); +void ValidateOptions(ZenStorageServerOptions& ServerOptions); + +struct ZenStorageServerCmdLineOptions +{ + // Note to those adding future options; std::filesystem::path-type options + // must be read into a std::string first. As of cxxopts-3.0.0 it uses a >> + // stream operator to convert argv value into the options type. std::fs::path + // expects paths in streams to be quoted but argv paths are unquoted. By + // going into a std::string first, paths with whitespace parse correctly. + std::string PluginsConfigFile; + std::string BaseSnapshotDir; + + void AddCliOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void ApplyOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string OpenIdProviderName; + std::string OpenIdProviderUrl; + std::string OpenIdClientId; + + void AddSecurityOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::string UpstreamCachePolicyOptions; + + void AddCacheOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + void AddGcOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + + std::vector<std::string> BucketConfigs; + + void AddObjectStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddBuildStoreOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); + void AddWorkspacesOptions(cxxopts::Options& options, ZenStorageServerOptions& ServerOptions); +}; + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstream.h b/src/zenserver/storage/upstream/upstream.h new file mode 100644 index 000000000..b4fd03983 --- /dev/null +++ b/src/zenserver/storage/upstream/upstream.h @@ -0,0 +1,7 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "upstreamcache.h" +#include "upstreamservice.h" +#include "zen.h" diff --git a/src/zenserver/storage/upstream/upstreamcache.cpp b/src/zenserver/storage/upstream/upstreamcache.cpp new file mode 100644 index 000000000..f7ae5f973 --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamcache.cpp @@ -0,0 +1,2134 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "upstreamcache.h" +#include "zen.h" + +#include <zencore/blockingqueue.h> +#include <zencore/compactbinary.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/fmtutils.h> +#include <zencore/stats.h> +#include <zencore/stream.h> +#include <zencore/timer.h> +#include <zencore/trace.h> + +#include <zenhttp/httpclientauth.h> +#include <zenhttp/packageformat.h> + +#include <zenstore/cache/structuredcachestore.h> +#include <zenstore/cidstore.h> + +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenremotestore/jupiter/jupitersession.h> + +#include "diag/logging.h" +#include "storage/cache/httpstructuredcache.h" + +#include <fmt/format.h> + +#include <algorithm> +#include <atomic> +#include <shared_mutex> +#include <thread> + +namespace zen { + +using namespace std::literals; + +namespace detail { + + class UpstreamStatus + { + public: + UpstreamEndpointState EndpointState() const { return static_cast<UpstreamEndpointState>(m_State.load(std::memory_order_relaxed)); } + + UpstreamEndpointStatus EndpointStatus() const + { + const UpstreamEndpointState State = EndpointState(); + { + std::unique_lock _(m_Mutex); + return {.Reason = m_ErrorText, .State = State}; + } + } + + void Set(UpstreamEndpointState NewState) + { + m_State.store(static_cast<uint32_t>(NewState), std::memory_order_relaxed); + { + std::unique_lock _(m_Mutex); + m_ErrorText.clear(); + } + } + + void Set(UpstreamEndpointState NewState, std::string ErrorText) + { + m_State.store(static_cast<uint32_t>(NewState), std::memory_order_relaxed); + { + std::unique_lock _(m_Mutex); + m_ErrorText = std::move(ErrorText); + } + } + + void SetFromErrorCode(int32_t ErrorCode, std::string_view ErrorText) + { + if (ErrorCode != 0) + { + Set(ErrorCode == 401 ? UpstreamEndpointState::kUnauthorized : UpstreamEndpointState::kError, std::string(ErrorText)); + } + } + + private: + mutable std::mutex m_Mutex; + std::string m_ErrorText; + std::atomic_uint32_t m_State; + }; + + class JupiterUpstreamEndpoint final : public UpstreamEndpoint + { + public: + JupiterUpstreamEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) + : m_AuthMgr(Mgr) + , m_Log(zen::logging::Get("upstream")) + { + ZEN_ASSERT(!Options.Name.empty()); + m_Info.Name = Options.Name; + m_Info.Url = Options.ServiceUrl; + + std::function<HttpClientAccessToken()> TokenProvider; + + if (AuthConfig.OAuthUrl.empty() == false) + { + TokenProvider = httpclientauth::CreateFromOAuthClientCredentials( + {.Url = AuthConfig.OAuthUrl, .ClientId = AuthConfig.OAuthClientId, .ClientSecret = AuthConfig.OAuthClientSecret}); + } + else if (!AuthConfig.OpenIdProvider.empty()) + { + TokenProvider = httpclientauth::CreateFromOpenIdProvider(m_AuthMgr, AuthConfig.OpenIdProvider); + } + else if (!AuthConfig.AccessToken.empty()) + { + TokenProvider = httpclientauth::CreateFromStaticToken(AuthConfig.AccessToken); + } + else + { + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(m_AuthMgr); + } + + m_Client = new JupiterClient(Options, std::move(TokenProvider)); + } + + virtual ~JupiterUpstreamEndpoint() {} + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } + + virtual UpstreamEndpointStatus Initialize() override + { + ZEN_TRACE_CPU("Upstream::Jupiter::Initialize"); + + try + { + if (m_Status.EndpointState() == UpstreamEndpointState::kOk) + { + return {.State = UpstreamEndpointState::kOk}; + } + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + const JupiterResult Result = Session.Authenticate(); + + if (Result.Success) + { + m_Status.Set(UpstreamEndpointState::kOk); + } + else if (Result.ErrorCode != 0) + { + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + } + else + { + m_Status.Set(UpstreamEndpointState::kUnauthorized); + } + + return m_Status.EndpointStatus(); + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = Err.what(), .State = GetState()}; + } + } + + std::string_view GetActualBlobStoreNamespace(std::string_view Namespace) + { + if (Namespace == ZenCacheStore::DefaultNamespace) + { + return m_Client->DefaultBlobStoreNamespace(); + } + return Namespace; + } + + virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } + + virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, + const CacheKey& CacheKey, + ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheRecord"); + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + JupiterResult Result; + + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + + if (Type == ZenContentType::kCompressedBinary) + { + Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + + if (Result.Success) + { + const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); + if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) + { + CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); + IoBuffer ContentBuffer; + int NumAttachments = 0; + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; + Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; + Result.ErrorCode = AttachmentResult.ErrorCode; + + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer::ValidateCompressedHeader(AttachmentResult.Response, RawHash, RawSize)) + { + Result.Response = AttachmentResult.Response; + ++NumAttachments; + } + else + { + Result.Success = false; + } + }); + if (NumAttachments != 1) + { + Result.Success = false; + } + } + } + } + else + { + const ZenContentType AcceptType = Type == ZenContentType::kCbPackage ? ZenContentType::kCbObject : Type; + Result = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, AcceptType); + + if (Result.Success && Type == ZenContentType::kCbPackage) + { + CbPackage Package; + + const CbValidateError ValidationResult = ValidateCompactBinary(Result.Response, CbValidateMode::All); + if (Result.Success = ValidationResult == CbValidateError::None; Result.Success) + { + CbObject CacheRecord = LoadCompactBinaryObject(Result.Response); + + CacheRecord.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult AttachmentResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + Result.ReceivedBytes += AttachmentResult.ReceivedBytes; + Result.SentBytes += AttachmentResult.SentBytes; + Result.ElapsedSeconds += AttachmentResult.ElapsedSeconds; + Result.ErrorCode = AttachmentResult.ErrorCode; + + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Chunk = + CompressedBuffer::FromCompressed(SharedBuffer(AttachmentResult.Response), RawHash, RawSize)) + { + Package.AddAttachment(CbAttachment(Chunk, AttachmentHash.AsHash())); + } + else + { + Result.Success = false; + } + }); + + Package.SetObject(CacheRecord); + } + + if (Result.Success) + { + BinaryWriter MemStream; + Package.Save(MemStream); + + Result.Response = IoBuffer(IoBuffer::Clone, MemStream.Data(), MemStream.Size()); + } + } + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span<CacheKeyRequest*> Requests, + OnCacheRecordGetComplete&& OnComplete) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheRecords"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheKeyRequest* Request : Requests) + { + const CacheKey& CacheKey = Request->Key; + CbPackage Package; + CbObject Record; + + double ElapsedSeconds = 0.0; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + JupiterResult RefResult = Session.GetRef(BlobStoreNamespace, CacheKey.Bucket, CacheKey.Hash, ZenContentType::kCbObject); + AppendResult(RefResult, Result); + ElapsedSeconds = RefResult.ElapsedSeconds; + + m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); + + if (RefResult.ErrorCode == 0) + { + const CbValidateError ValidationResult = ValidateCompactBinary(RefResult.Response, CbValidateMode::All); + if (ValidationResult == CbValidateError::None) + { + Record = LoadCompactBinaryObject(RefResult.Response); + Record.IterateAttachments([&](CbFieldView AttachmentHash) { + JupiterResult BlobResult = Session.GetCompressedBlob(BlobStoreNamespace, AttachmentHash.AsHash()); + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + + if (BlobResult.ErrorCode == 0) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer Chunk = + CompressedBuffer::FromCompressed(SharedBuffer(BlobResult.Response), RawHash, RawSize)) + { + if (RawHash == AttachmentHash.AsHash()) + { + Package.AddAttachment(CbAttachment(Chunk, RawHash)); + } + } + } + }); + } + } + } + + OnComplete( + {.Request = *Request, .Record = Record, .Package = Package, .ElapsedSeconds = ElapsedSeconds, .Source = &m_Info}); + } + + return Result; + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey&, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetSingleCacheChunk"); + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult Result = Session.GetCompressedBlob(BlobStoreNamespace, ValueContentId); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span<CacheChunkRequest*> CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheChunks"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + CacheChunkRequest& Request = *RequestPtr; + IoBuffer Payload; + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = 0; + + double ElapsedSeconds = 0.0; + bool IsCompressed = false; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const JupiterResult BlobResult = + Request.ChunkId == IoHash::Zero + ? Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, Request.ChunkId) + : Session.GetCompressedBlob(BlobStoreNamespace, Request.ChunkId); + ElapsedSeconds = BlobResult.ElapsedSeconds; + Payload = BlobResult.Response; + + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + if (Payload && IsCompressedBinary(Payload.GetContentType())) + { + IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize); + } + } + + if (IsCompressed) + { + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = Payload, + .ElapsedSeconds = ElapsedSeconds, + .Source = &m_Info}); + } + else + { + OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + return Result; + } + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span<CacheValueRequest*> CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Jupiter::GetCacheValues"); + + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + GetUpstreamCacheResult Result; + + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + CacheValueRequest& Request = *RequestPtr; + IoBuffer Payload; + IoHash RawHash = IoHash::Zero; + uint64_t RawSize = 0; + + double ElapsedSeconds = 0.0; + bool IsCompressed = false; + if (!Result.Error) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + IoHash PayloadHash; + const JupiterResult BlobResult = + Session.GetInlineBlob(BlobStoreNamespace, Request.Key.Bucket, Request.Key.Hash, PayloadHash); + ElapsedSeconds = BlobResult.ElapsedSeconds; + Payload = BlobResult.Response; + + AppendResult(BlobResult, Result); + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + if (Payload) + { + if (IsCompressedBinary(Payload.GetContentType())) + { + IsCompressed = CompressedBuffer::ValidateCompressedHeader(Payload, RawHash, RawSize) && RawHash != PayloadHash; + } + else + { + CompressedBuffer Compressed = CompressedBuffer::Compress(SharedBuffer(Payload)); + RawHash = Compressed.DecodeRawHash(); + if (RawHash == PayloadHash) + { + IsCompressed = true; + } + else + { + ZEN_WARN("Jupiter request for inline payload of {}/{}/{} has hash {}, expected hash {} from header", + Namespace, + Request.Key.Bucket, + Request.Key.Hash.ToHexString(), + RawHash.ToHexString(), + PayloadHash.ToHexString()); + } + } + } + } + + if (IsCompressed) + { + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = Payload, + .ElapsedSeconds = ElapsedSeconds, + .Source = &m_Info}); + } + else + { + OnComplete({.Request = Request, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + return Result; + } + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span<IoBuffer const> Values) override + { + ZEN_TRACE_CPU("Upstream::Jupiter::PutCacheRecord"); + + ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); + const int32_t MaxAttempts = 3; + + try + { + JupiterSession Session(m_Client->Logger(), m_Client->Client(), m_AllowRedirect); + + if (CacheRecord.Type == ZenContentType::kBinary) + { + JupiterResult Result; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(CacheRecord.Namespace); + Result = Session.PutRef(BlobStoreNamespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + RecordValue, + ZenContentType::kBinary); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + return {.Reason = std::move(Result.Reason), + .Bytes = gsl::narrow<int64_t>(Result.ReceivedBytes), + .ElapsedSeconds = Result.ElapsedSeconds, + .Success = Result.Success}; + } + else if (CacheRecord.Type == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + if (!CompressedBuffer::ValidateCompressedHeader(RecordValue, RawHash, RawSize)) + { + return {.Reason = std::string("Invalid compressed value buffer"), .Success = false}; + } + + CbObjectWriter ReferencingObject; + ReferencingObject.AddBinaryAttachment("RawHash", RawHash); + ReferencingObject.AddInteger("RawSize", RawSize); + + return PerformStructuredPut( + Session, + CacheRecord.Namespace, + CacheRecord.Key, + ReferencingObject.Save().GetBuffer().AsIoBuffer(), + MaxAttempts, + [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { + if (ValueContentId != RawHash) + { + OutReason = + fmt::format("Value '{}' MISMATCHED from compressed buffer raw hash {}", ValueContentId, RawHash); + return false; + } + + OutBuffer = RecordValue; + return true; + }); + } + else + { + return PerformStructuredPut( + Session, + CacheRecord.Namespace, + CacheRecord.Key, + RecordValue, + MaxAttempts, + [&](const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason) { + const auto It = + std::find(std::begin(CacheRecord.ValueContentIds), std::end(CacheRecord.ValueContentIds), ValueContentId); + + if (It == std::end(CacheRecord.ValueContentIds)) + { + OutReason = fmt::format("value '{}' MISSING from local cache", ValueContentId); + return false; + } + + const size_t Idx = std::distance(std::begin(CacheRecord.ValueContentIds), It); + + OutBuffer = Values[Idx]; + return true; + }); + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = std::string(Err.what()), .Success = false}; + } + } + + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + + private: + static void AppendResult(const JupiterResult& Result, GetUpstreamCacheResult& Out) + { + Out.Success &= Result.Success; + Out.Bytes += gsl::narrow<int64_t>(Result.ReceivedBytes); + Out.ElapsedSeconds += Result.ElapsedSeconds; + + if (Result.ErrorCode) + { + Out.Error = {.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}; + } + }; + + PutUpstreamCacheResult PerformStructuredPut( + JupiterSession& Session, + std::string_view Namespace, + const CacheKey& Key, + IoBuffer ObjectBuffer, + const int32_t MaxAttempts, + std::function<bool(const IoHash& ValueContentId, IoBuffer& OutBuffer, std::string& OutReason)>&& BlobFetchFn) + { + int64_t TotalBytes = 0ull; + double TotalElapsedSeconds = 0.0; + + std::string_view BlobStoreNamespace = GetActualBlobStoreNamespace(Namespace); + const auto PutBlobs = [&](std::span<IoHash> ValueContentIds, std::string& OutReason) -> bool { + for (const IoHash& ValueContentId : ValueContentIds) + { + IoBuffer BlobBuffer; + if (!BlobFetchFn(ValueContentId, BlobBuffer, OutReason)) + { + return false; + } + + JupiterResult BlobResult; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !BlobResult.Success; Attempt++) + { + BlobResult = Session.PutCompressedBlob(BlobStoreNamespace, ValueContentId, BlobBuffer); + } + + m_Status.SetFromErrorCode(BlobResult.ErrorCode, BlobResult.Reason); + + if (!BlobResult.Success) + { + OutReason = fmt::format("upload value '{}' FAILED, reason '{}'", ValueContentId, BlobResult.Reason); + return false; + } + + TotalBytes += gsl::narrow<int64_t>(BlobResult.ReceivedBytes); + TotalElapsedSeconds += BlobResult.ElapsedSeconds; + } + + return true; + }; + + PutRefResult RefResult; + for (int32_t Attempt = 0; Attempt < MaxAttempts && !RefResult.Success; Attempt++) + { + RefResult = Session.PutRef(BlobStoreNamespace, Key.Bucket, Key.Hash, ObjectBuffer, ZenContentType::kCbObject); + } + + m_Status.SetFromErrorCode(RefResult.ErrorCode, RefResult.Reason); + + if (!RefResult.Success) + { + return {.Reason = fmt::format("upload cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, RefResult.Reason), + .Success = false}; + } + + TotalBytes += gsl::narrow<int64_t>(RefResult.ReceivedBytes); + TotalElapsedSeconds += RefResult.ElapsedSeconds; + + std::string Reason; + if (!PutBlobs(RefResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } + + const IoHash RefHash = IoHash::HashBuffer(ObjectBuffer); + FinalizeRefResult FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); + + m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); + + if (!FinalizeResult.Success) + { + return { + .Reason = fmt::format("finalize cache record '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), + .Success = false}; + } + + if (!FinalizeResult.Needs.empty()) + { + if (!PutBlobs(FinalizeResult.Needs, Reason)) + { + return {.Reason = std::move(Reason), .Success = false}; + } + + FinalizeResult = Session.FinalizeRef(BlobStoreNamespace, Key.Bucket, Key.Hash, RefHash); + + m_Status.SetFromErrorCode(FinalizeResult.ErrorCode, FinalizeResult.Reason); + + if (!FinalizeResult.Success) + { + return {.Reason = fmt::format("finalize '{}/{}' FAILED, reason '{}'", Key.Bucket, Key.Hash, FinalizeResult.Reason), + .Success = false}; + } + + if (!FinalizeResult.Needs.empty()) + { + ExtendableStringBuilder<256> Sb; + for (const IoHash& MissingHash : FinalizeResult.Needs) + { + Sb << MissingHash.ToHexString() << ","; + } + + return { + .Reason = fmt::format("finalize '{}/{}' FAILED, still needs value(s) '{}'", Key.Bucket, Key.Hash, Sb.ToString()), + .Success = false}; + } + } + + TotalBytes += gsl::narrow<int64_t>(FinalizeResult.ReceivedBytes); + TotalElapsedSeconds += FinalizeResult.ElapsedSeconds; + + return {.Bytes = TotalBytes, .ElapsedSeconds = TotalElapsedSeconds, .Success = true}; + } + + LoggerRef Log() { return m_Log; } + + AuthMgr& m_AuthMgr; + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + RefPtr<JupiterClient> m_Client; + const bool m_AllowRedirect = false; + }; + + class ZenUpstreamEndpoint final : public UpstreamEndpoint + { + struct ZenEndpoint + { + std::string Url; + std::string Reason; + double Latency{}; + bool Ok = false; + + bool operator<(const ZenEndpoint& RHS) const { return Ok && RHS.Ok ? Latency < RHS.Latency : Ok; } + }; + + public: + ZenUpstreamEndpoint(const ZenStructuredCacheClientOptions& Options) + : m_Log(zen::logging::Get("upstream")) + , m_ConnectTimeout(Options.ConnectTimeout) + , m_Timeout(Options.Timeout) + { + ZEN_ASSERT(!Options.Name.empty()); + m_Info.Name = Options.Name; + + for (const auto& Url : Options.Urls) + { + m_Endpoints.push_back({.Url = Url}); + } + } + + ~ZenUpstreamEndpoint() {} + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const override { return m_Info; } + + virtual UpstreamEndpointStatus Initialize() override + { + ZEN_TRACE_CPU("Upstream::Zen::Initialize"); + + try + { + if (m_Status.EndpointState() == UpstreamEndpointState::kOk) + { + return {.State = UpstreamEndpointState::kOk}; + } + + const ZenEndpoint& Ep = GetEndpoint(); + + if (m_Info.Url != Ep.Url) + { + ZEN_INFO("Setting Zen upstream URL to '{}'", Ep.Url); + m_Info.Url = Ep.Url; + } + + if (Ep.Ok) + { + RwLock::ExclusiveLockScope _(m_ClientLock); + m_Client = new ZenStructuredCacheClient({.Url = m_Info.Url, .ConnectTimeout = m_ConnectTimeout, .Timeout = m_Timeout}); + m_Status.Set(UpstreamEndpointState::kOk); + } + else + { + m_Status.Set(UpstreamEndpointState::kError, Ep.Reason); + } + + return m_Status.EndpointStatus(); + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = Err.what(), .State = GetState()}; + } + } + + virtual UpstreamEndpointState GetState() override { return m_Status.EndpointState(); } + + virtual UpstreamEndpointStatus GetStatus() override { return m_Status.EndpointStatus(); } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, + const CacheKey& CacheKey, + ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetSingleCacheRecord"); + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + const ZenCacheResult Result = Session.GetCacheRecord(Namespace, CacheKey.Bucket, CacheKey.Hash, Type); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span<CacheKeyRequest*> Requests, + OnCacheRecordGetComplete&& OnComplete) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheRecords"); + ZEN_ASSERT(Requests.size() > 0); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheRecords"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = Requests[0]->Policy.GetRecordPolicy(); + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy); + + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("Requests"sv); + for (CacheKeyRequest* Request : Requests) + { + BatchRequest.BeginObject(); + { + const CacheKey& Key = Request->Key; + BatchRequest.BeginObject("Key"sv); + { + BatchRequest << "Bucket"sv << Key.Bucket; + BatchRequest << "Hash"sv << Key.Hash; + } + BatchRequest.EndObject(); + if (!Request->Policy.IsUniform() || Request->Policy.GetRecordPolicy() != DefaultPolicy) + { + BatchRequest.SetName("Policy"sv); + Request->Policy.Save(BatchRequest); + } + } + BatchRequest.EndObject(); + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (Results.Num() != Requests.size()) + { + ZEN_WARN("Upstream::Zen::GetCacheRecords invalid number of Response results from Upstream."); + } + else + { + for (size_t Index = 0; CbFieldView Record : Results) + { + CacheKeyRequest* Request = Requests[Index++]; + OnComplete({.Request = *Request, + .Record = Record.AsObjectView(), + .Package = BatchResponse, + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheRecords invalid Response from Upstream."); + } + } + + for (CacheKeyRequest* Request : Requests) + { + OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunk"); + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + const ZenCacheResult Result = Session.GetCacheChunk(Namespace, CacheKey.Bucket, CacheKey.Hash, ValueContentId); + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.ErrorCode == 0) + { + return {.Status = {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = Result.Success}, + .Value = Result.Response, + .Source = &m_Info}; + } + else + { + return {.Status = {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}}; + } + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Status = {.Error{.ErrorCode = -1, .Reason = Err.what()}}}; + } + } + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span<CacheValueRequest*> CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheValues"); + ZEN_ASSERT(!CacheValueRequests.empty()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheValues"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = CacheValueRequests[0]->Policy; + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("Requests"sv); + { + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + const CacheValueRequest& Request = *RequestPtr; + + BatchRequest.BeginObject(); + { + BatchRequest.BeginObject("Key"sv); + BatchRequest << "Bucket"sv << Request.Key.Bucket; + BatchRequest << "Hash"sv << Request.Key.Hash; + BatchRequest.EndObject(); + if (Request.Policy != DefaultPolicy) + { + BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); + } + } + BatchRequest.EndObject(); + } + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (CacheValueRequests.size() != Results.Num()) + { + ZEN_WARN("Upstream::Zen::GetCacheValues invalid number of Response results from Upstream."); + } + else + { + for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) + { + CacheValueRequest& Request = *CacheValueRequests[RequestIndex++]; + CbObjectView ChunkObject = ChunkField.AsObjectView(); + IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); + IoBuffer Payload; + uint64_t RawSize = 0; + if (RawHash != IoHash::Zero) + { + bool Success = false; + const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); + if (Attachment) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + RawSize = Compressed.DecodeRawSize(); + Success = true; + } + } + if (!Success) + { + CbFieldView RawSizeField = ChunkObject["RawSize"sv]; + RawSize = RawSizeField.AsUInt64(); + Success = !RawSizeField.HasError(); + } + if (!Success) + { + RawHash = IoHash::Zero; + } + } + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = std::move(Payload), + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheValues invalid Response from Upstream."); + } + } + + for (CacheValueRequest* RequestPtr : CacheValueRequests) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span<CacheChunkRequest*> CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::Zen::GetCacheChunks"); + ZEN_ASSERT(!CacheChunkRequests.empty()); + + CbObjectWriter BatchRequest; + BatchRequest << "Method"sv + << "GetCacheChunks"sv; + BatchRequest << "Accept"sv << kCbPkgMagic; + + BatchRequest.BeginObject("Params"sv); + { + CachePolicy DefaultPolicy = CacheChunkRequests[0]->Policy; + BatchRequest << "DefaultPolicy"sv << WriteToString<128>(DefaultPolicy).ToView(); + BatchRequest << "Namespace"sv << Namespace; + + BatchRequest.BeginArray("ChunkRequests"sv); + { + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + const CacheChunkRequest& Request = *RequestPtr; + + BatchRequest.BeginObject(); + { + BatchRequest.BeginObject("Key"sv); + BatchRequest << "Bucket"sv << Request.Key.Bucket; + BatchRequest << "Hash"sv << Request.Key.Hash; + BatchRequest.EndObject(); + if (Request.ValueId) + { + BatchRequest.AddObjectId("ValueId"sv, Request.ValueId); + } + if (Request.ChunkId != Request.ChunkId.Zero) + { + BatchRequest << "ChunkId"sv << Request.ChunkId; + } + if (Request.RawOffset != 0) + { + BatchRequest << "RawOffset"sv << Request.RawOffset; + } + if (Request.RawSize != UINT64_MAX) + { + BatchRequest << "RawSize"sv << Request.RawSize; + } + if (Request.Policy != DefaultPolicy) + { + BatchRequest << "Policy"sv << WriteToString<128>(Request.Policy).ToView(); + } + } + BatchRequest.EndObject(); + } + } + BatchRequest.EndArray(); + } + BatchRequest.EndObject(); + + ZenCacheResult Result; + + { + ZenStructuredCacheSession Session(GetClientRef()); + Result = Session.InvokeRpc(BatchRequest.Save()); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + if (Result.Success) + { + CbPackage BatchResponse; + if (ParsePackageMessageWithLegacyFallback(Result.Response, BatchResponse)) + { + CbArrayView Results = BatchResponse.GetObject()["Result"sv].AsArrayView(); + if (CacheChunkRequests.size() != Results.Num()) + { + ZEN_WARN("Upstream::Zen::GetCacheChunks invalid number of Response results from Upstream."); + } + else + { + for (size_t RequestIndex = 0; CbFieldView ChunkField : Results) + { + CacheChunkRequest& Request = *CacheChunkRequests[RequestIndex++]; + CbObjectView ChunkObject = ChunkField.AsObjectView(); + IoHash RawHash = ChunkObject["RawHash"sv].AsHash(); + IoBuffer Payload; + uint64_t RawSize = 0; + if (RawHash != IoHash::Zero) + { + bool Success = false; + const CbAttachment* Attachment = BatchResponse.FindAttachment(RawHash); + if (Attachment) + { + if (const CompressedBuffer& Compressed = Attachment->AsCompressedBinary()) + { + Payload = Compressed.GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + RawSize = Compressed.DecodeRawSize(); + Success = true; + } + } + if (!Success) + { + CbFieldView RawSizeField = ChunkObject["RawSize"sv]; + RawSize = RawSizeField.AsUInt64(); + Success = !RawSizeField.HasError(); + } + if (!Success) + { + RawHash = IoHash::Zero; + } + } + OnComplete({.Request = Request, + .RawHash = RawHash, + .RawSize = RawSize, + .Value = std::move(Payload), + .ElapsedSeconds = Result.ElapsedSeconds, + .Source = &m_Info}); + } + + return {.Bytes = Result.Bytes, .ElapsedSeconds = Result.ElapsedSeconds, .Success = true}; + } + } + else + { + ZEN_WARN("Upstream::Zen::GetCacheChunks invalid Response from Upstream."); + } + } + + for (CacheChunkRequest* RequestPtr : CacheChunkRequests) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + + return {.Error{.ErrorCode = Result.ErrorCode, .Reason = std::move(Result.Reason)}}; + } + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span<IoBuffer const> Values) override + { + ZEN_TRACE_CPU("Upstream::Zen::PutCacheRecord"); + + ZEN_ASSERT(CacheRecord.ValueContentIds.size() == Values.size()); + const int32_t MaxAttempts = 3; + + try + { + ZenStructuredCacheSession Session(GetClientRef()); + ZenCacheResult Result; + int64_t TotalBytes = 0ull; + double TotalElapsedSeconds = 0.0; + + if (CacheRecord.Type == ZenContentType::kCbPackage) + { + CbPackage Package; + Package.SetObject(CbObject(SharedBuffer(RecordValue))); + + for (const IoBuffer& Value : Values) + { + IoHash RawHash; + uint64_t RawSize; + if (CompressedBuffer AttachmentBuffer = CompressedBuffer::FromCompressed(SharedBuffer(Value), RawHash, RawSize)) + { + Package.AddAttachment(CbAttachment(AttachmentBuffer, RawHash)); + } + else + { + return {.Reason = std::string("Invalid value buffer"), .Success = false}; + } + } + + BinaryWriter MemStream; + Package.Save(MemStream); + IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); + + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheRecord(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + PackagePayload, + CacheRecord.Type); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes = Result.Bytes; + TotalElapsedSeconds = Result.ElapsedSeconds; + } + else if (CacheRecord.Type == ZenContentType::kCompressedBinary) + { + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(RecordValue), RawHash, RawSize); + if (!Compressed) + { + return {.Reason = std::string("Invalid value compressed buffer"), .Success = false}; + } + + CbPackage BatchPackage; + CbObjectWriter BatchWriter; + BatchWriter << "Method"sv + << "PutCacheValues"sv; + BatchWriter << "Accept"sv << kCbPkgMagic; + + BatchWriter.BeginObject("Params"sv); + { + // DefaultPolicy unspecified and expected to be Default + + BatchWriter << "Namespace"sv << CacheRecord.Namespace; + + BatchWriter.BeginArray("Requests"sv); + { + BatchWriter.BeginObject(); + { + const CacheKey& Key = CacheRecord.Key; + BatchWriter.BeginObject("Key"sv); + { + BatchWriter << "Bucket"sv << Key.Bucket; + BatchWriter << "Hash"sv << Key.Hash; + } + BatchWriter.EndObject(); + // Policy unspecified and expected to be Default + BatchWriter.AddBinaryAttachment("RawHash"sv, RawHash); + BatchPackage.AddAttachment(CbAttachment(Compressed, RawHash)); + } + BatchWriter.EndObject(); + } + BatchWriter.EndArray(); + } + BatchWriter.EndObject(); + BatchPackage.SetObject(BatchWriter.Save()); + + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.InvokeRpc(BatchPackage); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + } + else + { + for (size_t Idx = 0, Count = Values.size(); Idx < Count; Idx++) + { + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheValue(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + CacheRecord.ValueContentIds[Idx], + Values[Idx]); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + + if (!Result.Success) + { + return {.Reason = "Failed to upload value", + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = false}; + } + } + + Result.Success = false; + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + { + Result = Session.PutCacheRecord(CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + RecordValue, + CacheRecord.Type); + } + + m_Status.SetFromErrorCode(Result.ErrorCode, Result.Reason); + + TotalBytes += Result.Bytes; + TotalElapsedSeconds += Result.ElapsedSeconds; + } + + return {.Reason = std::move(Result.Reason), + .Bytes = TotalBytes, + .ElapsedSeconds = TotalElapsedSeconds, + .Success = Result.Success}; + } + catch (const std::exception& Err) + { + m_Status.Set(UpstreamEndpointState::kError, Err.what()); + + return {.Reason = std::string(Err.what()), .Success = false}; + } + } + + virtual UpstreamEndpointStats& Stats() override { return m_Stats; } + + private: + Ref<ZenStructuredCacheClient> GetClientRef() + { + // m_Client can be modified at any time by a different thread. + // Make sure we safely bump the refcount inside a scope lock + RwLock::SharedLockScope _(m_ClientLock); + ZEN_ASSERT(m_Client); + Ref<ZenStructuredCacheClient> ClientRef(m_Client); + _.ReleaseNow(); + return ClientRef; + } + + const ZenEndpoint& GetEndpoint() + { + for (ZenEndpoint& Ep : m_Endpoints) + { + Ref<ZenStructuredCacheClient> Client( + new ZenStructuredCacheClient({.Url = Ep.Url, .ConnectTimeout = std::chrono::milliseconds(1000)})); + ZenStructuredCacheSession Session(std::move(Client)); + const int32_t SampleCount = 2; + + Ep.Ok = false; + Ep.Latency = {}; + + for (int32_t Sample = 0; Sample < SampleCount; ++Sample) + { + ZenCacheResult Result = Session.CheckHealth(); + Ep.Ok = Result.Success; + Ep.Reason = std::move(Result.Reason); + Ep.Latency += Result.ElapsedSeconds; + } + Ep.Latency /= double(SampleCount); + } + + std::sort(std::begin(m_Endpoints), std::end(m_Endpoints)); + + for (const auto& Ep : m_Endpoints) + { + ZEN_INFO("ping 'Zen' endpoint '{}' latency '{:.3}s' {}", Ep.Url, Ep.Latency, Ep.Ok ? "OK" : Ep.Reason); + } + + return m_Endpoints.front(); + } + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + UpstreamEndpointInfo m_Info; + UpstreamStatus m_Status; + UpstreamEndpointStats m_Stats; + std::vector<ZenEndpoint> m_Endpoints; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; + RwLock m_ClientLock; + RefPtr<ZenStructuredCacheClient> m_Client; + }; + +} // namespace detail + +////////////////////////////////////////////////////////////////////////// + +class UpstreamCacheImpl final : public UpstreamCache +{ +public: + UpstreamCacheImpl(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) + : m_Log(logging::Get("upstream")) + , m_Options(Options) + , m_CacheStore(CacheStore) + , m_CidStore(CidStore) + { + } + + virtual ~UpstreamCacheImpl() { Shutdown(); } + + virtual void Initialize() override + { + ZEN_TRACE_CPU("Upstream::Initialize"); + + m_RunState.IsRunning = true; + } + + virtual bool IsActive() override + { + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + return !m_Endpoints.empty(); + } + + virtual void RegisterEndpoint(std::unique_ptr<UpstreamEndpoint> Endpoint) override + { + ZEN_TRACE_CPU("Upstream::RegisterEndpoint"); + + const UpstreamEndpointStatus Status = Endpoint->Initialize(); + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); + + if (Status.State == UpstreamEndpointState::kOk) + { + ZEN_INFO("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); + } + else + { + ZEN_WARN("register endpoint '{} - {}' {}", Info.Name, Info.Url, ToString(Status.State)); + } + + // Register endpoint even if it fails, the health monitor thread will probe failing endpoint(s) + std::unique_lock<std::shared_mutex> _(m_EndpointsMutex); + if (m_Endpoints.empty()) + { + for (uint32_t Idx = 0; Idx < m_Options.ThreadCount; Idx++) + { + m_UpstreamThreads.emplace_back(&UpstreamCacheImpl::ProcessUpstreamQueue, this, Idx + 1); + } + + m_EndpointMonitorThread = std::thread(&UpstreamCacheImpl::MonitorEndpoints, this); + } + m_Endpoints.emplace_back(std::move(Endpoint)); + } + + virtual void IterateEndpoints(std::function<bool(UpstreamEndpoint&)>&& Fn) override + { + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + for (auto& Ep : m_Endpoints) + { + if (!Fn(*Ep)) + { + break; + } + } + } + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) override + { + ZEN_TRACE_CPU("Upstream::GetCacheRecord"); + + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + GetUpstreamCacheSingleResult Result = Endpoint->GetCacheRecord(Namespace, CacheKey, Type); + Scope.Stop(); + + Stats.CacheGetCount.Increment(1); + Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); + + if (Result.Status.Success) + { + Stats.CacheHitCount.Increment(1); + + return Result; + } + + if (Result.Status.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache record FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Status.Error.Reason, + Result.Status.Error.ErrorCode); + } + } + } + + return {}; + } + + virtual void GetCacheRecords(std::string_view Namespace, + std::span<CacheKeyRequest*> Requests, + OnCacheRecordGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheRecords"); + + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + std::vector<CacheKeyRequest*> RemainingKeys(Requests.begin(), Requests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector<CacheKeyRequest*> Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + + Result = Endpoint->GetCacheRecords(Namespace, RemainingKeys, [&](CacheRecordGetCompleteParams&& Params) { + if (Params.Record) + { + OnComplete(std::forward<CacheRecordGetCompleteParams>(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache record(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheKeyRequest* Request : RemainingKeys) + { + OnComplete({.Request = *Request, .Record = CbObjectView(), .Package = CbPackage()}); + } + } + + virtual void GetCacheChunks(std::string_view Namespace, + std::span<CacheChunkRequest*> CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheChunks"); + + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + std::vector<CacheChunkRequest*> RemainingKeys(CacheChunkRequests.begin(), CacheChunkRequests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector<CacheChunkRequest*> Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); + + Result = Endpoint->GetCacheChunks(Namespace, RemainingKeys, [&](CacheChunkGetCompleteParams&& Params) { + if (Params.RawHash != Params.RawHash.Zero) + { + OnComplete(std::forward<CacheChunkGetCompleteParams>(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache chunks(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheChunkRequest* RequestPtr : RemainingKeys) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) override + { + ZEN_TRACE_CPU("Upstream::GetCacheChunk"); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + metrics::OperationTiming::Scope Scope(Stats.CacheGetRequestTiming); + GetUpstreamCacheSingleResult Result = Endpoint->GetCacheChunk(Namespace, CacheKey, ValueContentId); + Scope.Stop(); + + Stats.CacheGetCount.Increment(1); + Stats.CacheGetTotalBytes.Increment(Result.Status.Bytes); + + if (Result.Status.Success) + { + Stats.CacheHitCount.Increment(1); + + return Result; + } + + if (Result.Status.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache chunk FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Status.Error.Reason, + Result.Status.Error.ErrorCode); + } + } + } + + return {}; + } + + virtual void GetCacheValues(std::string_view Namespace, + std::span<CacheValueRequest*> CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) override final + { + ZEN_TRACE_CPU("Upstream::GetCacheValues"); + + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + std::vector<CacheValueRequest*> RemainingKeys(CacheValueRequests.begin(), CacheValueRequests.end()); + + if (m_Options.ReadUpstream) + { + for (auto& Endpoint : m_Endpoints) + { + if (RemainingKeys.empty()) + { + break; + } + + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + std::vector<CacheValueRequest*> Missing; + GetUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Endpoint->Stats().CacheGetRequestTiming); + + Result = Endpoint->GetCacheValues(Namespace, RemainingKeys, [&](CacheValueGetCompleteParams&& Params) { + if (Params.RawHash != Params.RawHash.Zero) + { + OnComplete(std::forward<CacheValueGetCompleteParams>(Params)); + + Stats.CacheHitCount.Increment(1); + } + else + { + Missing.push_back(&Params.Request); + } + }); + } + + Stats.CacheGetCount.Increment(int64_t(RemainingKeys.size())); + Stats.CacheGetTotalBytes.Increment(Result.Bytes); + + if (Result.Error) + { + Stats.CacheErrorCount.Increment(1); + + ZEN_WARN("get cache values(s) (rpc) FAILED, endpoint '{}', reason '{}', error code '{}'", + Endpoint->GetEndpointInfo().Url, + Result.Error.Reason, + Result.Error.ErrorCode); + } + + RemainingKeys = std::move(Missing); + } + } + + const UpstreamEndpointInfo Info; + for (CacheValueRequest* RequestPtr : RemainingKeys) + { + OnComplete({.Request = *RequestPtr, .RawHash = IoHash::Zero, .RawSize = 0, .Value = IoBuffer()}); + } + } + + virtual void EnqueueUpstream(UpstreamCacheRecord CacheRecord) override + { + if (m_RunState.IsRunning && m_Options.WriteUpstream && m_Endpoints.size() > 0) + { + if (!m_UpstreamThreads.empty()) + { + m_UpstreamQueue.Enqueue(std::move(CacheRecord)); + } + else + { + ProcessCacheRecord(std::move(CacheRecord)); + } + } + } + + virtual void GetStatus(CbObjectWriter& Status) override + { + ZEN_TRACE_CPU("Upstream::GetStatus"); + + Status << "active" << IsActive(); + Status << "reading" << m_Options.ReadUpstream; + Status << "writing" << m_Options.WriteUpstream; + Status << "worker_threads" << m_UpstreamThreads.size(); + Status << "queue_count" << m_UpstreamQueue.Size(); + + Status.BeginArray("endpoints"); + for (const auto& Ep : m_Endpoints) + { + const UpstreamEndpointInfo& EpInfo = Ep->GetEndpointInfo(); + const UpstreamEndpointStatus EpStatus = Ep->GetStatus(); + UpstreamEndpointStats& EpStats = Ep->Stats(); + + Status.BeginObject(); + Status << "name" << EpInfo.Name; + Status << "url" << EpInfo.Url; + Status << "state" << ToString(EpStatus.State); + Status << "reason" << EpStatus.Reason; + + Status.BeginObject("cache"sv); + { + const int64_t GetCount = EpStats.CacheGetCount.Value(); + const int64_t HitCount = EpStats.CacheHitCount.Value(); + const int64_t ErrorCount = EpStats.CacheErrorCount.Value(); + const double HitRatio = GetCount > 0 ? double(HitCount) / double(GetCount) : 0.0; + const double ErrorRatio = GetCount > 0 ? double(ErrorCount) / double(GetCount) : 0.0; + + metrics::EmitSnapshot("get_requests"sv, EpStats.CacheGetRequestTiming, Status); + Status << "get_bytes" << EpStats.CacheGetTotalBytes.Value(); + Status << "get_count" << GetCount; + Status << "hit_count" << HitCount; + Status << "hit_ratio" << HitRatio; + Status << "error_count" << ErrorCount; + Status << "error_ratio" << ErrorRatio; + metrics::EmitSnapshot("put_requests"sv, EpStats.CachePutRequestTiming, Status); + Status << "put_bytes" << EpStats.CachePutTotalBytes.Value(); + } + Status.EndObject(); + + Status.EndObject(); + } + Status.EndArray(); + } + +private: + void ProcessCacheRecord(UpstreamCacheRecord CacheRecord) + { + ZEN_TRACE_CPU("Upstream::ProcessCacheRecord"); + + ZenCacheValue CacheValue; + std::vector<IoBuffer> Payloads; + + if (!m_CacheStore.Get(CacheRecord.Context, CacheRecord.Namespace, CacheRecord.Key.Bucket, CacheRecord.Key.Hash, CacheValue)) + { + ZEN_WARN("process upstream FAILED, '{}/{}/{}', cache record doesn't exist", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash); + return; + } + + for (const IoHash& ValueContentId : CacheRecord.ValueContentIds) + { + if (IoBuffer Payload = m_CidStore.FindChunkByCid(ValueContentId)) + { + Payloads.push_back(Payload); + } + else + { + ZEN_WARN("process upstream FAILED, '{}/{}/{}/{}', ValueContentId doesn't exist in CAS", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + ValueContentId); + return; + } + } + + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + for (auto& Endpoint : m_Endpoints) + { + if (Endpoint->GetState() != UpstreamEndpointState::kOk) + { + continue; + } + + UpstreamEndpointStats& Stats = Endpoint->Stats(); + PutUpstreamCacheResult Result; + { + metrics::OperationTiming::Scope Scope(Stats.CachePutRequestTiming); + Result = Endpoint->PutCacheRecord(CacheRecord, CacheValue.Value, std::span(Payloads)); + } + + Stats.CachePutTotalBytes.Increment(Result.Bytes); + + if (!Result.Success) + { + ZEN_WARN("upload cache record '{}/{}/{}' FAILED, endpoint '{}', reason '{}'", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + Endpoint->GetEndpointInfo().Url, + Result.Reason); + } + } + } + + void ProcessUpstreamQueue(int ThreadIndex) + { + std::string ThreadName = fmt::format("upstream_{}", ThreadIndex); + SetCurrentThreadName(ThreadName); + + for (;;) + { + UpstreamCacheRecord CacheRecord; + if (m_UpstreamQueue.WaitAndDequeue(CacheRecord)) + { + try + { + ProcessCacheRecord(std::move(CacheRecord)); + } + catch (const std::exception& Err) + { + ZEN_ERROR("upload cache record '{}/{}/{}' FAILED, reason '{}'", + CacheRecord.Namespace, + CacheRecord.Key.Bucket, + CacheRecord.Key.Hash, + Err.what()); + } + } + + if (!m_RunState.IsRunning) + { + break; + } + } + } + + void MonitorEndpoints() + { + SetCurrentThreadName("upstream_monitor"); + + for (;;) + { + { + std::unique_lock lk(m_RunState.Mutex); + if (m_RunState.ExitSignal.wait_for(lk, m_Options.HealthCheckInterval, [this]() { return !m_RunState.IsRunning.load(); })) + { + break; + } + } + + try + { + std::vector<UpstreamEndpoint*> Endpoints; + + { + std::shared_lock<std::shared_mutex> _(m_EndpointsMutex); + + for (auto& Endpoint : m_Endpoints) + { + UpstreamEndpointState State = Endpoint->GetState(); + if (State == UpstreamEndpointState::kError) + { + Endpoints.push_back(Endpoint.get()); + ZEN_WARN("HEALTH - endpoint '{} - {}' is in error state '{}'", + Endpoint->GetEndpointInfo().Name, + Endpoint->GetEndpointInfo().Url, + Endpoint->GetStatus().Reason); + } + if (State == UpstreamEndpointState::kUnauthorized) + { + Endpoints.push_back(Endpoint.get()); + } + } + } + + for (auto& Endpoint : Endpoints) + { + const UpstreamEndpointInfo& Info = Endpoint->GetEndpointInfo(); + const UpstreamEndpointStatus Status = Endpoint->Initialize(); + + if (Status.State == UpstreamEndpointState::kOk) + { + ZEN_INFO("HEALTH - endpoint '{} - {}' Ok", Info.Name, Info.Url); + } + else + { + const std::string Reason = Status.Reason.empty() ? "" : fmt::format(", reason '{}'", Status.Reason); + ZEN_WARN("HEALTH - endpoint '{} - {}' {} {}", Info.Name, Info.Url, ToString(Status.State), Reason); + } + } + } + catch (const std::exception& Err) + { + ZEN_ERROR("check endpoint(s) health FAILED, reason '{}'", Err.what()); + } + } + } + + void Shutdown() + { + if (m_RunState.Stop()) + { + m_UpstreamQueue.CompleteAdding(); + for (std::thread& Thread : m_UpstreamThreads) + { + Thread.join(); + } + if (m_EndpointMonitorThread.joinable()) + { + m_EndpointMonitorThread.join(); + } + m_UpstreamThreads.clear(); + m_Endpoints.clear(); + } + } + + LoggerRef Log() { return m_Log; } + + using UpstreamQueue = BlockingQueue<UpstreamCacheRecord>; + + struct RunState + { + std::mutex Mutex; + std::condition_variable ExitSignal; + std::atomic_bool IsRunning{false}; + + bool Stop() + { + bool Stopped = false; + { + std::lock_guard _(Mutex); + Stopped = IsRunning.exchange(false); + } + if (Stopped) + { + ExitSignal.notify_all(); + } + return Stopped; + } + }; + + LoggerRef m_Log; + UpstreamCacheOptions m_Options; + ZenCacheStore& m_CacheStore; + CidStore& m_CidStore; + UpstreamQueue m_UpstreamQueue; + std::shared_mutex m_EndpointsMutex; + std::vector<std::unique_ptr<UpstreamEndpoint>> m_Endpoints; + std::vector<std::thread> m_UpstreamThreads; + std::thread m_EndpointMonitorThread; + RunState m_RunState; +}; + +////////////////////////////////////////////////////////////////////////// + +std::unique_ptr<UpstreamEndpoint> +UpstreamEndpoint::CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options) +{ + return std::make_unique<detail::ZenUpstreamEndpoint>(Options); +} + +std::unique_ptr<UpstreamEndpoint> +UpstreamEndpoint::CreateJupiterEndpoint(const JupiterClientOptions& Options, const UpstreamAuthConfig& AuthConfig, AuthMgr& Mgr) +{ + return std::make_unique<detail::JupiterUpstreamEndpoint>(Options, AuthConfig, Mgr); +} + +std::unique_ptr<UpstreamCache> +CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore) +{ + return std::make_unique<UpstreamCacheImpl>(Options, CacheStore, CidStore); +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamcache.h b/src/zenserver/storage/upstream/upstreamcache.h new file mode 100644 index 000000000..d5d61c8d9 --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamcache.h @@ -0,0 +1,167 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/compactbinary.h> +#include <zencore/compress.h> +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/stats.h> +#include <zencore/zencore.h> +#include <zenstore/cache/cache.h> +#include <zenstore/cache/upstreamcacheclient.h> + +#include <atomic> +#include <chrono> +#include <functional> +#include <memory> +#include <vector> + +namespace zen { + +class CbObjectView; +class AuthMgr; +class CbObjectView; +class CbPackage; +class CbObjectWriter; +class CidStore; +class ZenCacheStore; +struct JupiterClientOptions; +class JupiterAccessTokenProvider; +struct ZenStructuredCacheClientOptions; + +struct UpstreamEndpointStats +{ + metrics::OperationTiming CacheGetRequestTiming; + metrics::OperationTiming CachePutRequestTiming; + metrics::Counter CacheGetTotalBytes; + metrics::Counter CachePutTotalBytes; + metrics::Counter CacheGetCount; + metrics::Counter CacheHitCount; + metrics::Counter CacheErrorCount; +}; + +enum class UpstreamEndpointState : uint32_t +{ + kDisabled, + kUnauthorized, + kError, + kOk +}; + +inline std::string_view +ToString(UpstreamEndpointState State) +{ + using namespace std::literals; + + switch (State) + { + case UpstreamEndpointState::kDisabled: + return "Disabled"sv; + case UpstreamEndpointState::kUnauthorized: + return "Unauthorized"sv; + case UpstreamEndpointState::kError: + return "Error"sv; + case UpstreamEndpointState::kOk: + return "Ok"sv; + default: + return "Unknown"sv; + } +} + +struct UpstreamAuthConfig +{ + std::string_view OAuthUrl; + std::string_view OAuthClientId; + std::string_view OAuthClientSecret; + std::string_view OpenIdProvider; + std::string_view AccessToken; +}; + +struct UpstreamEndpointStatus +{ + std::string Reason; + UpstreamEndpointState State; +}; + +struct GetUpstreamCacheSingleResult +{ + GetUpstreamCacheResult Status; + IoBuffer Value; + const UpstreamEndpointInfo* Source = nullptr; +}; + +/** + * The upstream endpoint is responsible for handling upload/downloading of cache records. + */ +class UpstreamEndpoint +{ +public: + virtual ~UpstreamEndpoint() = default; + + virtual UpstreamEndpointStatus Initialize() = 0; + + virtual const UpstreamEndpointInfo& GetEndpointInfo() const = 0; + + virtual UpstreamEndpointState GetState() = 0; + virtual UpstreamEndpointStatus GetStatus() = 0; + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; + virtual GetUpstreamCacheResult GetCacheRecords(std::string_view Namespace, + std::span<CacheKeyRequest*> Requests, + OnCacheRecordGetComplete&& OnComplete) = 0; + + virtual GetUpstreamCacheResult GetCacheValues(std::string_view Namespace, + std::span<CacheValueRequest*> CacheValueRequests, + OnCacheValueGetComplete&& OnComplete) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, const CacheKey& CacheKey, const IoHash& PayloadId) = 0; + virtual GetUpstreamCacheResult GetCacheChunks(std::string_view Namespace, + std::span<CacheChunkRequest*> CacheChunkRequests, + OnCacheChunksGetComplete&& OnComplete) = 0; + + virtual PutUpstreamCacheResult PutCacheRecord(const UpstreamCacheRecord& CacheRecord, + IoBuffer RecordValue, + std::span<IoBuffer const> Payloads) = 0; + + virtual UpstreamEndpointStats& Stats() = 0; + + static std::unique_ptr<UpstreamEndpoint> CreateZenEndpoint(const ZenStructuredCacheClientOptions& Options); + + static std::unique_ptr<UpstreamEndpoint> CreateJupiterEndpoint(const JupiterClientOptions& Options, + const UpstreamAuthConfig& AuthConfig, + AuthMgr& Mgr); +}; + +/** + * Manages one or more upstream cache endpoints. + */ + +class UpstreamCache : public UpstreamCacheClient +{ +public: + virtual void Initialize() = 0; + + virtual void RegisterEndpoint(std::unique_ptr<UpstreamEndpoint> Endpoint) = 0; + virtual void IterateEndpoints(std::function<bool(UpstreamEndpoint&)>&& Fn) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheRecord(std::string_view Namespace, const CacheKey& CacheKey, ZenContentType Type) = 0; + + virtual GetUpstreamCacheSingleResult GetCacheChunk(std::string_view Namespace, + const CacheKey& CacheKey, + const IoHash& ValueContentId) = 0; + + virtual void GetStatus(CbObjectWriter& CbO) = 0; +}; + +struct UpstreamCacheOptions +{ + std::chrono::seconds HealthCheckInterval{5}; + uint32_t ThreadCount = 4; + bool ReadUpstream = true; + bool WriteUpstream = true; +}; + +std::unique_ptr<UpstreamCache> CreateUpstreamCache(const UpstreamCacheOptions& Options, ZenCacheStore& CacheStore, CidStore& CidStore); + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamservice.cpp b/src/zenserver/storage/upstream/upstreamservice.cpp new file mode 100644 index 000000000..e7f7d2d5c --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamservice.cpp @@ -0,0 +1,55 @@ +// Copyright Epic Games, Inc. All Rights Reserved. +#include "upstreamservice.h" + +#include "upstreamcache.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/string.h> + +namespace zen { + +using namespace std::literals; + +HttpUpstreamService::HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr) : m_Upstream(Upstream), m_AuthMgr(Mgr) +{ + m_Router.RegisterRoute( + "endpoints", + [this](HttpRouterRequest& Req) { + CbObjectWriter Writer; + Writer.BeginArray("Endpoints"sv); + m_Upstream.IterateEndpoints([&Writer](UpstreamEndpoint& Ep) { + UpstreamEndpointInfo Info = Ep.GetEndpointInfo(); + UpstreamEndpointStatus Status = Ep.GetStatus(); + + Writer.BeginObject(); + Writer << "Name"sv << Info.Name; + Writer << "Url"sv << Info.Url; + Writer << "State"sv << ToString(Status.State); + Writer << "Reason"sv << Status.Reason; + Writer.EndObject(); + + return true; + }); + Writer.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Writer.Save()); + }, + HttpVerb::kGet); +} + +HttpUpstreamService::~HttpUpstreamService() +{ +} + +const char* +HttpUpstreamService::BaseUri() const +{ + return "/upstream/"; +} + +void +HttpUpstreamService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/upstreamservice.h b/src/zenserver/storage/upstream/upstreamservice.h new file mode 100644 index 000000000..f1da03c8c --- /dev/null +++ b/src/zenserver/storage/upstream/upstreamservice.h @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenhttp/httpserver.h> + +namespace zen { + +class AuthMgr; +class UpstreamCache; + +class HttpUpstreamService final : public zen::HttpService +{ +public: + HttpUpstreamService(UpstreamCache& Upstream, AuthMgr& Mgr); + virtual ~HttpUpstreamService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + UpstreamCache& m_Upstream; + AuthMgr& m_AuthMgr; + HttpRequestRouter m_Router; +}; + +} // namespace zen diff --git a/src/zenserver/storage/upstream/zen.cpp b/src/zenserver/storage/upstream/zen.cpp new file mode 100644 index 000000000..25fd3a3bb --- /dev/null +++ b/src/zenserver/storage/upstream/zen.cpp @@ -0,0 +1,251 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zen.h" + +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/fmtutils.h> +#include <zencore/session.h> +#include <zencore/stream.h> +#include <zenhttp/formatters.h> +#include <zenhttp/httpclient.h> +#include <zenhttp/httpcommon.h> +#include <zenhttp/packageformat.h> + +#include <zenstore/cache/structuredcachestore.h> +#include "diag/logging.h" + +#include <xxhash.h> +#include <gsl/gsl-lite.hpp> + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +ZenStructuredCacheClient::ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options) +: m_Log(logging::Get(std::string_view("zenclient"))) +, m_ServiceUrl(Options.Url) +, m_ConnectTimeout(Options.ConnectTimeout) +, m_Timeout(Options.Timeout) +{ +} + +ZenStructuredCacheClient::~ZenStructuredCacheClient() +{ +} + +////////////////////////////////////////////////////////////////////////// + +using namespace std::literals; + +ZenStructuredCacheSession::ZenStructuredCacheSession(Ref<ZenStructuredCacheClient>&& OuterClient) +: m_Log(OuterClient->Log()) +, m_Client(std::move(OuterClient)) +{ +} + +ZenStructuredCacheSession::~ZenStructuredCacheSession() +{ +} + +ZenCacheResult +ZenStructuredCacheSession::CheckHealth() +{ + HttpClient Http{m_Client->ServiceUrl()}; + + HttpClient::Response Response = Http.Get("/health/check"sv); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + return {.Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Response.StatusCode == HttpResponseCode::OK}; +} + +ZenCacheResult +ZenStructuredCacheSession::GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString(); + + HttpClient::Response Response = Http.Get(Uri, {{"Accept", std::string{MapContentTypeToString(Type)}}}); + ZEN_DEBUG("GET {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::GetCacheChunk(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); + + HttpClient::Response Response = Http.Get(Uri, {{"Accept", "application/x-ue-comp"}}); + ZEN_DEBUG("GET {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = Buffer, .Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::PutCacheRecord(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + IoBuffer Value, + ZenContentType Type) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString(); + + Value.SetContentType(Type); + + HttpClient::Response Response = Http.Put(Uri, Value); + ZEN_DEBUG("PUT {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::PutCacheValue(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId, + IoBuffer Payload) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/"; + if (Namespace != ZenCacheStore::DefaultNamespace) + { + Uri << Namespace << "/"; + } + Uri << BucketId << "/" << Key.ToHexString() << "/" << ValueContentId.ToHexString(); + + Payload.SetContentType(HttpContentType::kCompressedBinary); + + HttpClient::Response Response = Http.Put(Uri, Payload); + ZEN_DEBUG("PUT {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK || Response.StatusCode == HttpResponseCode::Created; + + return {.Bytes = Response.DownloadedBytes, .ElapsedSeconds = Response.ElapsedSeconds, .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::InvokeRpc(const CbObjectView& Request) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/$rpc"; + + // TODO: this seems redundant, we should be able to send the data more directly, without the BinaryWriter + + BinaryWriter BodyWriter; + Request.CopyTo(BodyWriter); + + IoBuffer Body{IoBuffer::Wrap, BodyWriter.GetData(), BodyWriter.GetSize()}; + Body.SetContentType(HttpContentType::kCbObject); + + HttpClient::Response Response = Http.Post(Uri, Body, {{"Accept", "application/x-ue-cbpkg"}}); + ZEN_DEBUG("POST {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = std::move(Buffer), + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Success}; +} + +ZenCacheResult +ZenStructuredCacheSession::InvokeRpc(const CbPackage& Request) +{ + HttpClient Http{m_Client->ServiceUrl()}; + + ExtendableStringBuilder<256> Uri; + Uri << "/z$/$rpc"; + + IoBuffer Message = FormatPackageMessageBuffer(Request).Flatten().AsIoBuffer(); + Message.SetContentType(HttpContentType::kCbPackage); + + HttpClient::Response Response = Http.Post(Uri, Message, {{"Accept", "application/x-ue-cbpkg"}}); + ZEN_DEBUG("POST {}", Response); + + if (auto& Error = Response.Error; Error) + { + return {.ErrorCode = static_cast<int32_t>(Error->ErrorCode), .Reason = std::move(Error->ErrorMessage)}; + } + + const bool Success = Response.StatusCode == HttpResponseCode::OK; + const IoBuffer Buffer = Success ? Response.ResponsePayload : IoBuffer{}; + + return {.Response = std::move(Buffer), + .Bytes = Response.DownloadedBytes, + .ElapsedSeconds = Response.ElapsedSeconds, + .Success = Success}; +} + +} // namespace zen diff --git a/src/zenserver/storage/upstream/zen.h b/src/zenserver/storage/upstream/zen.h new file mode 100644 index 000000000..6321b46b1 --- /dev/null +++ b/src/zenserver/storage/upstream/zen.h @@ -0,0 +1,103 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/iobuffer.h> +#include <zencore/iohash.h> +#include <zencore/logging.h> +#include <zencore/memoryview.h> +#include <zencore/uid.h> +#include <zencore/zencore.h> + +#include <chrono> + +struct ZenCacheValue; + +namespace zen { + +class CbObjectWriter; +class CbObjectView; +class CbPackage; +class ZenStructuredCacheClient; + +////////////////////////////////////////////////////////////////////////// + +struct ZenCacheResult +{ + IoBuffer Response; + int64_t Bytes = {}; + double ElapsedSeconds = {}; + int32_t ErrorCode = {}; + std::string Reason; + bool Success = false; +}; + +struct ZenStructuredCacheClientOptions +{ + std::string_view Name; + std::string_view Url; + std::span<std::string const> Urls; + std::chrono::milliseconds ConnectTimeout{}; + std::chrono::milliseconds Timeout{}; +}; + +/** Zen Structured Cache session + * + * This provides a context in which cache queries can be performed + * + * These are currently all synchronous. Will need to be made asynchronous + */ +class ZenStructuredCacheSession +{ +public: + ZenStructuredCacheSession(Ref<ZenStructuredCacheClient>&& OuterClient); + ~ZenStructuredCacheSession(); + + ZenCacheResult CheckHealth(); + ZenCacheResult GetCacheRecord(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, ZenContentType Type); + ZenCacheResult GetCacheChunk(std::string_view Namespace, std::string_view BucketId, const IoHash& Key, const IoHash& ValueContentId); + ZenCacheResult PutCacheRecord(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + IoBuffer Value, + ZenContentType Type); + ZenCacheResult PutCacheValue(std::string_view Namespace, + std::string_view BucketId, + const IoHash& Key, + const IoHash& ValueContentId, + IoBuffer Payload); + ZenCacheResult InvokeRpc(const CbObjectView& Request); + ZenCacheResult InvokeRpc(const CbPackage& Package); + +private: + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + Ref<ZenStructuredCacheClient> m_Client; +}; + +/** Zen Structured Cache client + * + * This represents an endpoint to query -- actual queries should be done via + * ZenStructuredCacheSession + */ +class ZenStructuredCacheClient : public RefCounted +{ +public: + ZenStructuredCacheClient(const ZenStructuredCacheClientOptions& Options); + ~ZenStructuredCacheClient(); + + std::string_view ServiceUrl() const { return m_ServiceUrl; } + + inline LoggerRef Log() { return m_Log; } + +private: + LoggerRef m_Log; + std::string m_ServiceUrl; + std::chrono::milliseconds m_ConnectTimeout; + std::chrono::milliseconds m_Timeout; + + friend class ZenStructuredCacheSession; +}; + +} // namespace zen diff --git a/src/zenserver/storage/vfs/vfsservice.cpp b/src/zenserver/storage/vfs/vfsservice.cpp new file mode 100644 index 000000000..863ec348a --- /dev/null +++ b/src/zenserver/storage/vfs/vfsservice.cpp @@ -0,0 +1,179 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "vfsservice.h" + +#include <zenstore/vfsimpl.h> + +#include <zencore/compactbinarybuilder.h> + +namespace zen { + +using namespace std::literals; + +#if ZEN_WITH_VFS + +////////////////////////////////////////////////////////////////////////// + +bool +GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) +{ + IoBuffer Payload = HttpReq.ReadPayload(); + HttpContentType PayloadContentType = HttpReq.RequestContentType(); + + switch (PayloadContentType) + { + case HttpContentType::kJSON: + case HttpContentType::kUnknownContentType: + case HttpContentType::kText: + { + std::string JsonText(reinterpret_cast<const char*>(Payload.GetData()), Payload.GetSize()); + Cb = LoadCompactBinaryFromJson(JsonText).AsObject(); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected JSON format"); + return false; + } + } + break; + case HttpContentType::kCbObject: + Cb = LoadCompactBinaryObject(Payload); + if (!Cb) + { + HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Content format not supported, expected compact binary format"); + return false; + } + break; + default: + HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid request content type"); + return false; + } + + return true; +} + +////////////////////////////////////////////////////////////////////////// +// +// to test: +// +// echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- +// echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- + +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl) +{ + m_Router.RegisterRoute( + "info", + [&](HttpRouterRequest& Request) { + CbObjectWriter Cbo; + Cbo << "running" << m_Impl->IsVfsRunning(); + Cbo << "rootpath" << m_Impl->GetMountpointPath(); + + Request.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "", + [&](HttpRouterRequest& Req) { + CbObject Payload; + + if (!GetContentAsCbObject(Req.ServerRequest(), Payload)) + return; + + std::string_view RpcName = Payload["method"sv].AsString(); + + if (RpcName == "mount"sv) + { + CbObjectView Params = Payload["params"sv].AsObjectView(); + std::string_view Mountpath = Params["path"sv].AsString(); + + if (Mountpath.empty()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "no path specified"); + } + + if (m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS already mounted"); + } + + try + { + m_Impl->Mount(Mountpath); + } + catch (const std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else if (RpcName == "unmount"sv) + { + if (!m_Impl->IsVfsRunning()) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "VFS not active"); + } + + try + { + m_Impl->Unmount(); + } + catch (const std::exception& Ex) + { + return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, Ex.what()); + } + + Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "unknown RPC"sv); + } + }, + HttpVerb::kPost); + m_StatusService.RegisterHandler("vfs", *this); +} + +VfsService::~VfsService() +{ + m_StatusService.UnregisterHandler("vfs", *this); +} + +#else + +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService) +{ + ZEN_UNUSED(ServiceImpl); +} + +VfsService::~VfsService() +{ +} + +#endif + +const char* +VfsService::BaseUri() const +{ + return "/vfs/"; +} + +void +VfsService::HandleStatusRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +VfsService::HandleRequest(HttpServerRequest& HttpServiceRequest) +{ + m_Router.HandleRequest(HttpServiceRequest); +} + +} // namespace zen diff --git a/src/zenserver/storage/vfs/vfsservice.h b/src/zenserver/storage/vfs/vfsservice.h new file mode 100644 index 000000000..4e06da878 --- /dev/null +++ b/src/zenserver/storage/vfs/vfsservice.h @@ -0,0 +1,48 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zenbase/refcount.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstatus.h> + +#include <memory> + +namespace zen { + +class ProjectStore; +class ZenCacheStore; +struct VfsServiceImpl; + +/** Virtual File System service + + Implements support for exposing data via a virtual file system interface. Currently + this is primarily used to surface various data stored in the local storage service + to users for debugging and exploration purposes. + + Currently, it surfaces information from the structured cache service and from the + project store. + + */ + +class VfsService : public HttpService, public IHttpStatusProvider +{ +public: + explicit VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl); + ~VfsService(); + +protected: + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + VfsServiceImpl* m_Impl = nullptr; + + HttpStatusService& m_StatusService; + HttpRequestRouter m_Router; + + friend struct VfsServiceDataSource; +}; + +} // namespace zen diff --git a/src/zenserver/storage/workspaces/httpworkspaces.cpp b/src/zenserver/storage/workspaces/httpworkspaces.cpp new file mode 100644 index 000000000..3fea46b2f --- /dev/null +++ b/src/zenserver/storage/workspaces/httpworkspaces.cpp @@ -0,0 +1,1211 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "httpworkspaces.h" + +#include <zencore/basicfile.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenstore/workspaces.h> +#include <zenutil/chunkrequests.h> +#include <zenutil/workerpools.h> + +#include <unordered_set> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogFs, "fs"sv); + +namespace { + + std::filesystem::path GetPathParameter(HttpServerRequest& ServerRequest, std::string_view Name) + { + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue(Name); !Value.empty()) + { + return std::filesystem::path(HttpServerRequest::Decode(Value)); + } + return {}; + } + + void WriteWorkspaceConfig(CbWriter& Writer, const Workspaces::WorkspaceConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "root_path" << Config.RootPath.string(); // utf8? + Writer << "allow_share_creation_from_http" << Config.AllowShareCreationFromHttp; + }; + + void WriteWorkspaceShareConfig(CbWriter& Writer, const Workspaces::WorkspaceShareConfiguration& Config) + { + Writer << "id" << Config.Id; + Writer << "share_path" << Config.SharePath.string(); // utf8? + if (!Config.Alias.empty()) + { + Writer << "alias" << Config.Alias; + } + }; + + void WriteWorkspaceAndSharesConfig(CbWriter& Writer, Workspaces& Workspaces, const Workspaces::WorkspaceConfiguration& WorkspaceConfig) + { + WriteWorkspaceConfig(Writer, WorkspaceConfig); + if (std::optional<std::vector<Oid>> ShareIds = Workspaces.GetWorkspaceShares(WorkspaceConfig.Id); ShareIds) + { + Writer.BeginArray("shares"); + { + for (const Oid& ShareId : *ShareIds) + { + if (std::optional<Workspaces::WorkspaceShareConfiguration> WorkspaceShareConfig = + Workspaces.GetWorkspaceShareConfiguration(WorkspaceConfig.Id, ShareId); + WorkspaceShareConfig) + { + Writer.BeginObject(); + { + WriteWorkspaceShareConfig(Writer, *WorkspaceShareConfig); + } + Writer.EndObject(); + } + } + } + Writer.EndArray(); + } + } + +} // namespace + +HttpWorkspacesService::HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces) +: m_Log(logging::Get("workspaces")) +, m_StatusService(StatusService) +, m_StatsService(StatsService) +, m_Config(Cfg) +, m_Workspaces(Workspaces) +{ + Initialize(); +} + +HttpWorkspacesService::~HttpWorkspacesService() +{ + m_StatsService.UnregisterHandler("ws", *this); + m_StatusService.UnregisterHandler("ws", *this); +} + +const char* +HttpWorkspacesService::BaseUri() const +{ + return "/ws/"; +} + +void +HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogFs, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("WorkspacesService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("workspaces"); + { + Cbo.BeginObject("workspace"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceReadCount << "writecount" << m_WorkspacesStats.WorkspaceWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("workspaceshare"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceShareReadCount << "writecount" << m_WorkspacesStats.WorkspaceShareWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceShareDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_WorkspacesStats.WorkspaceShareChunkHitCount << "misscount" + << m_WorkspacesStats.WorkspaceShareChunkMissCount; + } + Cbo.EndObject(); + + Cbo << "filescount" << m_WorkspacesStats.WorkspaceShareFilesReadCount; + Cbo << "entriescount" << m_WorkspacesStats.WorkspaceShareEntriesReadCount; + Cbo << "batchcount" << m_WorkspacesStats.WorkspaceShareBatchReadCount; + + Cbo << "requestcount" << m_WorkspacesStats.RequestCount; + Cbo << "badrequestcount" << m_WorkspacesStats.BadRequestCount; + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::HandleStatusRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpWorkspacesService::Status"); + CbObjectWriter Cbo; + Cbo << "ok" << true; + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::Initialize() +{ + using namespace std::literals; + + ZEN_LOG_INFO(LogFs, "Initializing Workspaces Service"); + + m_Router.AddPattern("workspace_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_alias", "([[:alnum:]_.\\+\\-\\[\\]]+)"); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/files", + [this](HttpRouterRequest& Req) { FilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}/info", + [this](HttpRouterRequest& Req) { ChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/batch", + [this](HttpRouterRequest& Req) { BatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/entries", + [this](HttpRouterRequest& Req) { EntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}", + [this](HttpRouterRequest& Req) { ChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}/files", + [this](HttpRouterRequest& Req) { ShareAliasFilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}/info", + [this](HttpRouterRequest& Req) { ShareAliasChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/batch", + [this](HttpRouterRequest& Req) { ShareAliasBatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "share/{share_alias}/entries", + [this](HttpRouterRequest& Req) { ShareAliasEntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "share/{share_alias}/{chunk}", + [this](HttpRouterRequest& Req) { ShareAliasChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "share/{share_alias}", + [this](HttpRouterRequest& Req) { ShareAliasRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}", + [this](HttpRouterRequest& Req) { ShareRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}", + [this](HttpRouterRequest& Req) { WorkspaceRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "refresh", + [this](HttpRouterRequest& Req) { RefreshRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "", + [this](HttpRouterRequest& Req) { WorkspacesRequest(Req); }, + HttpVerb::kGet); + + RefreshState(); + + m_StatsService.RegisterHandler("ws", *this); + m_StatusService.RegisterHandler("ws", *this); +} + +std::filesystem::path +HttpWorkspacesService::GetStatePath() const +{ + return m_Config.SystemRootDir / "workspaces"; +} + +void +HttpWorkspacesService::RefreshState() +{ + if (!m_Config.SystemRootDir.empty()) + { + m_Workspaces.RefreshState(GetStatePath()); + } +} + +bool +HttpWorkspacesService::MayChangeConfiguration(const HttpServerRequest& Req) const +{ + ZEN_UNUSED(Req); + return m_Config.AllowConfigurationChanges; +} + +void +HttpWorkspacesService::RefreshRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); +} + +void +HttpWorkspacesService::WorkspacesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + std::vector<Oid> WorkspaceIds = m_Workspaces.GetWorkspaces(); + CbObjectWriter Response; + Response.BeginArray("workspaces"); + for (const Oid& WorkspaceId : WorkspaceIds) + { + if (std::optional<Workspaces::WorkspaceConfiguration> WorkspaceConfig = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + WorkspaceConfig) + { + Response.BeginObject(); + { + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *WorkspaceConfig); + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + FilesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkInfoRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + BatchRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + EntriesRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + ChunkRequest(Req, WorkspaceId, ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + Oid ShareId = Oid::Zero; + if (Req.GetCapture(2) != Oid::Zero.ToString()) + { + ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + } + ShareRequest(Req, WorkspaceId, ShareId); +} + +void +HttpWorkspacesService::WorkspaceRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path WorkspacePath = GetPathParameter(ServerRequest, "root_path"sv); + if (WorkspacePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'root_path' parameter"); + } + + if (Req.GetCapture(1) == Oid::Zero.ToString()) + { + // Synthesize Id + WorkspaceId = Workspaces::PathToId(WorkspacePath); + ZEN_INFO("Generated workspace id from path '{}': {}", WorkspacePath, WorkspaceId); + } + else if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace {} is not allowed", WorkspaceId)); + } + bool AllowShareCreationFromHttp = false; + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue("allow_share_creation_from_http"); Value == "true"sv) + { + AllowShareCreationFromHttp = true; + } + + m_WorkspacesStats.WorkspaceWriteCount++; + Workspaces::WorkspaceConfiguration OldConfig = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + Workspaces::WorkspaceConfiguration NewConfig = {.Id = WorkspaceId, + .RootPath = WorkspacePath, + .AllowShareCreationFromHttp = AllowShareCreationFromHttp}; + if (OldConfig.Id == WorkspaceId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with root path '{}'", WorkspaceId, OldConfig.RootPath)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceConfiguration ConfigWithSameRoot = + Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspacePath); + ConfigWithSameRoot.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with same root path '{}'", ConfigWithSameRoot.Id, WorkspacePath)); + } + } + + bool Created = Workspaces::AddWorkspace(Log(), GetStatePath(), NewConfig); + if (Created) + { + ZEN_ASSERT(OldConfig.Id == Oid::Zero); + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceReadCount++; + std::optional<Workspaces::WorkspaceConfiguration> Workspace = m_Workspaces.GetWorkspaceConfiguration(WorkspaceId); + if (Workspace) + { + CbObjectWriter Response; + WriteWorkspaceAndSharesConfig(Response, m_Workspaces, *Workspace); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + else + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse(HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace {} is not allowed", WorkspaceId)); + } + + m_WorkspacesStats.WorkspaceDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +void +HttpWorkspacesService::ShareAliasFilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + FilesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkInfoRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasBatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + BatchRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasEntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + EntriesRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::ShareAliasChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(2))); + } + ChunkRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId, ChunkId); +} + +void +HttpWorkspacesService::ShareAliasRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view Alias = Req.GetCapture(1); + if (Alias.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid alias '{}'", Req.GetCapture(1))); + } + std::optional<Workspaces::ShareAlias> WorkspaceAndShareId = m_Workspaces.GetShareAlias(Alias); + if (!WorkspaceAndShareId.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + ShareRequest(Req, WorkspaceAndShareId.value().WorkspaceId, WorkspaceAndShareId.value().ShareId); +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + m_WorkspacesStats.WorkspaceShareFilesReadCount++; + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = ServerRequest.GetQueryParams().GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); + const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); + + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + Response.BeginArray("files"sv); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + if (WantsSizeField) + { + Response << "size"sv << Entry.Size; + } + if (WantsRawSizeField) + { + Response << "rawsize"sv << Entry.Size; + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Workspaces::ShareFile File = + m_Workspaces.GetWorkspaceShareChunkInfo(WorkspaceId, ShareId, ChunkId, GetSmallWorkerPool(EWorkloadType::Burst)); + if (File.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "size"sv << File.Size; + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + IoBuffer Payload = ServerRequest.ReadPayload(); + std::optional<std::vector<RequestChunkEntry>> ChunkRequests = ParseChunkBatchRequest(Payload); + if (!ChunkRequests.has_value()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); + } + m_WorkspacesStats.WorkspaceShareBatchReadCount++; + std::vector<Workspaces::ChunkRequest> Requests; + Requests.reserve(ChunkRequests.value().size()); + std::transform(ChunkRequests.value().begin(), + ChunkRequests.value().end(), + std::back_inserter(Requests), + [](const RequestChunkEntry& Entry) { + return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; + }); + std::vector<IoBuffer> Chunks = + m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool(EWorkloadType::Burst)); + if (Chunks.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + for (const IoBuffer& Buffer : Chunks) + { + if (Buffer) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + } + else + { + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + } + } + std::vector<IoBuffer> Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); + if (!Response.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); + } + return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + fmt::format("failed formatting response for batch of {} chunks", Chunks.size())); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view OpKey = ServerRequest.GetQueryParams().GetValue("opkey"sv); + if (!OpKey.empty() && OpKey != "file_manifest") + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldfilter")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + m_WorkspacesStats.WorkspaceShareEntriesReadCount++; + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + + CbObjectWriter Response; + + if (OpKey.empty()) + { + Response.BeginArray("entries"sv); + Response.BeginObject(); + } + else + { + Response.BeginObject("entry"sv); + } + { + // Synthesize a fake op + Response << "key" + << "file_manifest"; + + Response.BeginArray("files"); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + { + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + } + Response.EndObject(); + } + } + Response.EndArray(); + } + + if (OpKey.empty()) + { + Response.EndObject(); + Response.EndArray(); + } + else + { + Response.EndObject(); + } + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + if (auto OffsetParm = ServerRequest.GetQueryParams().GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid offset parameter '{}'", OffsetParm)); + } + } + + if (auto SizeParm = ServerRequest.GetQueryParams().GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt<uint64_t>(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid size parameter '{}'", SizeParm)); + } + } + + std::vector<IoBuffer> Response = m_Workspaces.GetWorkspaceShareChunks( + WorkspaceId, + ShareId, + std::vector<Workspaces::ChunkRequest>{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, + GetSmallWorkerPool(EWorkloadType::Burst)); + if (!Response.empty() && Response[0]) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + if (Response[0].GetSize() == 0) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response[0].GetContentType(), Response); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId) +{ + Oid ShareId = InShareId; + + HttpServerRequest& ServerRequest = Req.ServerRequest(); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path SharePath = GetPathParameter(ServerRequest, "share_path"sv); + if (SharePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'share_path' parameter"); + } + + if (ShareId == Oid::Zero) + { + // Synthesize Id + ShareId = Workspaces::PathToId(SharePath); + ZEN_INFO("Generated workspace id from path '{}': {}", SharePath, ShareId); + } + + std::string Alias = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("alias"sv)); + if (!AsciiSet::HasOnly(Alias, Workspaces::ValidAliasCharactersSet)) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid 'alias' parameter"); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Workspace '{}' does not exist", WorkspaceId)); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Adding workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareWriteCount++; + + const Workspaces::WorkspaceShareConfiguration OldConfig = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, ShareId); + const Workspaces::WorkspaceShareConfiguration NewConfig = {.Id = ShareId, + .SharePath = SharePath, + .Alias = std::string(Alias)}; + + if (OldConfig.Id == ShareId && (OldConfig != NewConfig)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + else if (OldConfig.Id == Oid::Zero) + { + if (Workspaces::WorkspaceShareConfiguration ConfigWithSamePath = + Workspaces::FindWorkspaceShare(Log(), Workspace.RootPath, SharePath); + ConfigWithSamePath.Id != Oid::Zero) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}' with same share path '{}' and alias '{}'", + ShareId, + WorkspaceId, + OldConfig.SharePath, + OldConfig.Alias)); + } + } + + if (!IsDir(Workspace.RootPath / NewConfig.SharePath)) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("directory {} does not exist in workspace {} root '{}'", + NewConfig.SharePath, + WorkspaceId, + Workspace.RootPath)); + } + + bool Created = Workspaces::AddWorkspaceShare(Log(), Workspace.RootPath, NewConfig); + if (Created) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", ShareId)); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", ShareId)); + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + m_WorkspacesStats.WorkspaceShareReadCount++; + std::optional<Workspaces::WorkspaceShareConfiguration> Config = + m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); + if (!Config) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + WriteWorkspaceShareConfig(Response, *Config); + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", ShareId)); + } + + Workspaces::WorkspaceConfiguration Workspace = Workspaces::FindWorkspace(Log(), GetStatePath(), WorkspaceId); + if (Workspace.Id == Oid::Zero) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + if (!Workspace.AllowShareCreationFromHttp) + { + if (!MayChangeConfiguration(ServerRequest)) + { + return ServerRequest.WriteResponse( + HttpResponseCode::Unauthorized, + HttpContentType::kText, + fmt::format("Removing workspace share {} in workspace {} is not allowed", WorkspaceId, ShareId)); + } + } + + m_WorkspacesStats.WorkspaceShareDeleteCount++; + bool Deleted = Workspaces::RemoveWorkspaceShare(Log(), Workspace.RootPath, ShareId); + if (Deleted) + { + RefreshState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +} // namespace zen diff --git a/src/zenserver/storage/workspaces/httpworkspaces.h b/src/zenserver/storage/workspaces/httpworkspaces.h new file mode 100644 index 000000000..89a8e8bdc --- /dev/null +++ b/src/zenserver/storage/workspaces/httpworkspaces.h @@ -0,0 +1,97 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> +#include <zenhttp/httpstatus.h> + +namespace zen { + +class Workspaces; + +struct WorkspacesServeConfig +{ + std::filesystem::path SystemRootDir; + bool AllowConfigurationChanges = false; +}; + +class HttpWorkspacesService final : public HttpService, public IHttpStatusProvider, public IHttpStatsProvider +{ +public: + HttpWorkspacesService(HttpStatusService& StatusService, + HttpStatsService& StatsService, + const WorkspacesServeConfig& Cfg, + Workspaces& Workspaces); + virtual ~HttpWorkspacesService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + struct WorkspacesStats + { + std::atomic_uint64_t WorkspaceReadCount{}; + std::atomic_uint64_t WorkspaceWriteCount{}; + std::atomic_uint64_t WorkspaceDeleteCount{}; + std::atomic_uint64_t WorkspaceShareReadCount{}; + std::atomic_uint64_t WorkspaceShareWriteCount{}; + std::atomic_uint64_t WorkspaceShareDeleteCount{}; + std::atomic_uint64_t WorkspaceShareFilesReadCount{}; + std::atomic_uint64_t WorkspaceShareEntriesReadCount{}; + std::atomic_uint64_t WorkspaceShareBatchReadCount{}; + std::atomic_uint64_t WorkspaceShareChunkHitCount{}; + std::atomic_uint64_t WorkspaceShareChunkMissCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void Initialize(); + std::filesystem::path GetStatePath() const; + void RefreshState(); + // void WriteState(); + + bool MayChangeConfiguration(const HttpServerRequest& Req) const; + + void WorkspacesRequest(HttpRouterRequest& Req); + void RefreshRequest(HttpRouterRequest& Req); + void FilesRequest(HttpRouterRequest& Req); + void ChunkInfoRequest(HttpRouterRequest& Req); + void BatchRequest(HttpRouterRequest& Req); + void EntriesRequest(HttpRouterRequest& Req); + void ChunkRequest(HttpRouterRequest& Req); + void ShareRequest(HttpRouterRequest& Req); + void WorkspaceRequest(HttpRouterRequest& Req); + + void ShareAliasFilesRequest(HttpRouterRequest& Req); + void ShareAliasChunkInfoRequest(HttpRouterRequest& Req); + void ShareAliasBatchRequest(HttpRouterRequest& Req); + void ShareAliasEntriesRequest(HttpRouterRequest& Req); + void ShareAliasChunkRequest(HttpRouterRequest& Req); + void ShareAliasRequest(HttpRouterRequest& Req); + + void FilesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkInfoRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void BatchRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void EntriesRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId); + void ChunkRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& ShareId, const Oid& ChunkId); + void ShareRequest(HttpRouterRequest& Req, const Oid& WorkspaceId, const Oid& InShareId); + + HttpStatusService& m_StatusService; + HttpStatsService& m_StatsService; + const WorkspacesServeConfig m_Config; + HttpRequestRouter m_Router; + Workspaces& m_Workspaces; + WorkspacesStats m_WorkspacesStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp new file mode 100644 index 000000000..73896512d --- /dev/null +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -0,0 +1,961 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenstorageserver.h" + +#include <zenbase/refcount.h> +#include <zencore/basicfile.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinaryvalidation.h> +#include <zencore/config.h> +#include <zencore/except.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zencore/iobuffer.h> +#include <zencore/jobqueue.h> +#include <zencore/logging.h> +#include <zencore/scopeguard.h> +#include <zencore/sentryintegration.h> +#include <zencore/session.h> +#include <zencore/string.h> +#include <zencore/thread.h> +#include <zencore/timer.h> +#include <zencore/trace.h> +#include <zencore/workthreadpool.h> +#include <zenhttp/httpserver.h> +#include <zenremotestore/jupiter/jupiterclient.h> +#include <zenstore/buildstore/buildstore.h> +#include <zenstore/cidstore.h> +#include <zenstore/scrubcontext.h> +#include <zenstore/vfsimpl.h> +#include <zenstore/workspaces.h> +#include <zenutil/service.h> +#include <zenutil/workerpools.h> +#include <zenutil/zenserverprocess.h> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + +#if ZEN_PLATFORM_LINUX +# include <pwd.h> +#endif + +#if ZEN_PLATFORM_MAC +# include <pwd.h> +#endif + +ZEN_THIRD_PARTY_INCLUDES_START +#include <fmt/format.h> +#include <asio.hpp> +ZEN_THIRD_PARTY_INCLUDES_END + +#include <exception> +#include <set> + +////////////////////////////////////////////////////////////////////////// + +#include "diag/logging.h" +#include "storageconfig.h" + +#include <zencore/memory/llm.h> + +namespace zen { + +namespace utils { + asio::error_code ResolveHostname(asio::io_context& Ctx, + std::string_view Host, + std::string_view DefaultPort, + std::vector<std::string>& OutEndpoints) + { + std::string_view Port = DefaultPort; + + if (const size_t Idx = Host.find(":"); Idx != std::string_view::npos) + { + Port = Host.substr(Idx + 1); + Host = Host.substr(0, Idx); + } + + asio::ip::tcp::resolver Resolver(Ctx); + + asio::error_code ErrorCode; + asio::ip::tcp::resolver::results_type Endpoints = Resolver.resolve(Host, Port, ErrorCode); + + if (!ErrorCode) + { + for (const asio::ip::tcp::endpoint Ep : Endpoints) + { + OutEndpoints.push_back(fmt::format("http://{}:{}", Ep.address().to_string(), Ep.port())); + } + } + + return ErrorCode; + } +} // namespace utils + +using namespace std::literals; + +ZenStorageServer::ZenStorageServer() +{ +} + +ZenStorageServer::~ZenStorageServer() +{ +} + +int +ZenStorageServer::Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry) +{ + ZEN_TRACE_CPU("ZenStorageServer::Initialize"); + ZEN_MEMSCOPE(GetZenserverTag()); + + const int EffectiveBasePort = ZenServerBase::Initialize(ServerOptions, ServerEntry); + if (EffectiveBasePort < 0) + { + return EffectiveBasePort; + } + + m_DebugOptionForcedCrash = ServerOptions.ShouldCrash; + m_StartupScrubOptions = ServerOptions.ScrubOptions; + + InitializeState(ServerOptions); + InitializeServices(ServerOptions); + RegisterServices(); + + ZenServerBase::Finalize(); + + return EffectiveBasePort; +} + +void +ZenStorageServer::RegisterServices() +{ + m_Http->RegisterService(*m_AuthService); + m_Http->RegisterService(m_StatsService); + m_Http->RegisterService(m_TestService); // NOTE: this is intentionally not limited to test mode as it's useful for diagnostics + +#if ZEN_WITH_TESTS + m_Http->RegisterService(m_TestingService); +#endif + + if (m_StructuredCacheService) + { + m_Http->RegisterService(*m_StructuredCacheService); + } + + if (m_UpstreamService) + { + m_Http->RegisterService(*m_UpstreamService); + } + + if (m_HttpProjectService) + { + m_Http->RegisterService(*m_HttpProjectService); + } + + if (m_HttpWorkspacesService) + { + m_Http->RegisterService(*m_HttpWorkspacesService); + } + + m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot, m_StatusService); + + if (m_FrontendService) + { + m_Http->RegisterService(*m_FrontendService); + } + + if (m_ObjStoreService) + { + m_Http->RegisterService(*m_ObjStoreService); + } + + if (m_BuildStoreService) + { + m_Http->RegisterService(*m_BuildStoreService); + } + +#if ZEN_WITH_VFS + m_Http->RegisterService(*m_VfsService); +#endif // ZEN_WITH_VFS + + m_Http->RegisterService(*m_AdminService); +} + +void +ZenStorageServer::InitializeServices(const ZenStorageServerOptions& ServerOptions) +{ + InitializeAuthentication(ServerOptions); + + ZEN_INFO("initializing storage"); + + CidStoreConfiguration Config; + Config.RootDirectory = m_DataRoot / "cas"; + + m_CidStore = std::make_unique<CidStore>(m_GcManager); + m_CidStore->Initialize(Config); + + ZEN_INFO("instantiating project service"); + + m_JobQueue = MakeJobQueue(8, "bgjobs"); + + m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, ProjectStore::Configuration{}); + m_HttpProjectService.reset( + new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatusService, m_StatsService, *m_AuthMgr, *m_OpenProcessCache, *m_JobQueue}); + + if (ServerOptions.WorksSpacesConfig.Enabled) + { + m_Workspaces.reset(new Workspaces()); + m_HttpWorkspacesService.reset( + new HttpWorkspacesService(m_StatusService, + m_StatsService, + {.SystemRootDir = ServerOptions.SystemRootDir, + .AllowConfigurationChanges = ServerOptions.WorksSpacesConfig.AllowConfigurationChanges}, + *m_Workspaces)); + } + + if (ServerOptions.BuildStoreConfig.Enabled) + { + CidStoreConfiguration BuildCidConfig; + BuildCidConfig.RootDirectory = m_DataRoot / "builds_cas"; + m_BuildCidStore = std::make_unique<CidStore>(m_GcManager); + m_BuildCidStore->Initialize(BuildCidConfig); + + BuildStoreConfig BuildsCfg; + BuildsCfg.RootDirectory = m_DataRoot / "builds"; + BuildsCfg.MaxDiskSpaceLimit = ServerOptions.BuildStoreConfig.MaxDiskSpaceLimit; + m_BuildStore = std::make_unique<BuildStore>(std::move(BuildsCfg), m_GcManager, *m_BuildCidStore); + } + + if (ServerOptions.StructuredCacheConfig.Enabled) + { + InitializeStructuredCache(ServerOptions); + } + else + { + ZEN_INFO("NOT instantiating structured cache service"); + } + + if (ServerOptions.ObjectStoreEnabled) + { + ObjectStoreConfig ObjCfg; + ObjCfg.RootDirectory = m_DataRoot / "obj"; + + for (const auto& Bucket : ServerOptions.ObjectStoreConfig.Buckets) + { + ObjectStoreConfig::BucketConfig NewBucket{.Name = Bucket.Name}; + NewBucket.Directory = Bucket.Directory.empty() ? (ObjCfg.RootDirectory / Bucket.Name) : Bucket.Directory; + ObjCfg.Buckets.push_back(std::move(NewBucket)); + } + + m_ObjStoreService = std::make_unique<HttpObjectStoreService>(m_StatusService, std::move(ObjCfg)); + } + + if (ServerOptions.BuildStoreConfig.Enabled) + { + m_BuildStoreService = std::make_unique<HttpBuildStoreService>(m_StatusService, m_StatsService, *m_BuildStore); + } + +#if ZEN_WITH_VFS + m_VfsServiceImpl = std::make_unique<VfsServiceImpl>(); + m_VfsServiceImpl->AddService(Ref<ProjectStore>(m_ProjectStore)); + m_VfsServiceImpl->AddService(Ref<ZenCacheStore>(m_CacheStore)); + + m_VfsService = std::make_unique<VfsService>(m_StatusService, m_VfsServiceImpl.get()); +#endif // ZEN_WITH_VFS + + ZEN_INFO("initializing GC, enabled '{}', interval {}, lightweight interval {}", + ServerOptions.GcConfig.Enabled, + NiceTimeSpanMs(ServerOptions.GcConfig.IntervalSeconds * 1000ull), + NiceTimeSpanMs(ServerOptions.GcConfig.LightweightIntervalSeconds * 1000ull)); + + GcSchedulerConfig GcConfig{.RootDirectory = m_DataRoot / "gc", + .MonitorInterval = std::chrono::seconds(ServerOptions.GcConfig.MonitorIntervalSeconds), + .Interval = std::chrono::seconds(ServerOptions.GcConfig.IntervalSeconds), + .MaxCacheDuration = std::chrono::seconds(ServerOptions.GcConfig.Cache.MaxDurationSeconds), + .MaxProjectStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.ProjectStore.MaxDurationSeconds), + .MaxBuildStoreDuration = std::chrono::seconds(ServerOptions.GcConfig.BuildStore.MaxDurationSeconds), + .CollectSmallObjects = ServerOptions.GcConfig.CollectSmallObjects, + .Enabled = ServerOptions.GcConfig.Enabled, + .DiskReserveSize = ServerOptions.GcConfig.DiskReserveSize, + .DiskSizeSoftLimit = ServerOptions.GcConfig.DiskSizeSoftLimit, + .MinimumFreeDiskSpaceToAllowWrites = ServerOptions.GcConfig.MinimumFreeDiskSpaceToAllowWrites, + .LightweightInterval = std::chrono::seconds(ServerOptions.GcConfig.LightweightIntervalSeconds), + .UseGCVersion = ServerOptions.GcConfig.UseGCV2 ? GcVersion::kV2 : GcVersion::kV1_Deprecated, + .CompactBlockUsageThresholdPercent = ServerOptions.GcConfig.CompactBlockUsageThresholdPercent, + .Verbose = ServerOptions.GcConfig.Verbose, + .SingleThreaded = ServerOptions.GcConfig.SingleThreaded, + .AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount}; + m_GcScheduler.Initialize(GcConfig); + + // Create and register admin interface last to make sure all is properly initialized + m_AdminService = std::make_unique<HttpAdminService>( + m_GcScheduler, + *m_JobQueue, + m_CacheStore.Get(), + [this]() { Flush(); }, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", + .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, + ServerOptions); +} + +void +ZenStorageServer::InitializeAuthentication(const ZenStorageServerOptions& ServerOptions) +{ + // Setup authentication manager + { + ZEN_TRACE_CPU("ZenStorageServer::InitAuth"); + std::string EncryptionKey = ServerOptions.EncryptionKey; + + if (EncryptionKey.empty()) + { + EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; + + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption key for authentication state"); + } + } + + std::string EncryptionIV = ServerOptions.EncryptionIV; + + if (EncryptionIV.empty()) + { + EncryptionIV = "0123456789abcdef"; + + if (ServerOptions.IsDedicated) + { + ZEN_WARN("Using default encryption initialization vector for authentication state"); + } + } + + m_AuthMgr = AuthMgr::Create({.RootDirectory = m_DataRoot / "auth", + .EncryptionKey = AesKey256Bit::FromString(EncryptionKey), + .EncryptionIV = AesIV128Bit::FromString(EncryptionIV)}); + + for (const ZenOpenIdProviderConfig& OpenIdProvider : ServerOptions.AuthConfig.OpenIdProviders) + { + m_AuthMgr->AddOpenIdProvider({.Name = OpenIdProvider.Name, .Url = OpenIdProvider.Url, .ClientId = OpenIdProvider.ClientId}); + } + } + + m_AuthService = std::make_unique<HttpAuthService>(*m_AuthMgr); +} + +void +ZenStorageServer::InitializeState(const ZenStorageServerOptions& ServerOptions) +{ + ZEN_TRACE_CPU("ZenStorageServer::InitializeState"); + + // Check root manifest to deal with schema versioning + + bool WipeState = false; + std::string WipeReason = "Unspecified"; + + if (ServerOptions.IsCleanStart) + { + WipeState = true; + WipeReason = "clean start requested"; + } + + bool UpdateManifest = false; + std::filesystem::path ManifestPath = m_DataRoot / "root_manifest"; + Oid StateId = Oid::Zero; + DateTime CreatedWhen{0}; + + if (!WipeState) + { + FileContents ManifestData = ReadFile(ManifestPath); + + if (ManifestData.ErrorCode) + { + if (ServerOptions.IsFirstRun) + { + ZEN_INFO("Initializing state at '{}'", m_DataRoot); + + UpdateManifest = true; + } + else + { + WipeState = true; + WipeReason = fmt::format("No manifest present at '{}'", ManifestPath); + } + } + else + { + IoBuffer Manifest = ManifestData.Flatten(); + + if (CbValidateError ValidationResult = ValidateCompactBinary(Manifest, CbValidateMode::All); + ValidationResult != CbValidateError::None) + { + ZEN_WARN("Manifest validation failed: {}, state will be wiped", zen::ToString(ValidationResult)); + + WipeState = true; + WipeReason = fmt::format("Validation of manifest at '{}' failed: {}", ManifestPath, zen::ToString(ValidationResult)); + } + else + { + m_RootManifest = LoadCompactBinaryObject(Manifest); + + const int32_t ManifestVersion = m_RootManifest["schema_version"].AsInt32(0); + StateId = m_RootManifest["state_id"].AsObjectId(); + CreatedWhen = m_RootManifest["created"].AsDateTime(); + + if (ManifestVersion != ZEN_CFG_SCHEMA_VERSION) + { + std::filesystem::path ManifestSkipSchemaChangePath = m_DataRoot / "root_manifest.ignore_schema_mismatch"; + if (ManifestVersion != 0 && IsFile(ManifestSkipSchemaChangePath)) + { + ZEN_INFO( + "Schema version {} found in '{}' does not match {}, ignoring mismatch due to existance of '{}' and updating " + "schema version", + ManifestVersion, + ManifestPath, + ZEN_CFG_SCHEMA_VERSION, + ManifestSkipSchemaChangePath); + UpdateManifest = true; + } + else + { + WipeState = true; + WipeReason = + fmt::format("Manifest schema version: {}, differs from required: {}", ManifestVersion, ZEN_CFG_SCHEMA_VERSION); + } + } + } + } + } + + if (StateId == Oid::Zero) + { + StateId = Oid::NewOid(); + UpdateManifest = true; + } + + const DateTime Now = DateTime::Now(); + + if (CreatedWhen.GetTicks() == 0) + { + CreatedWhen = Now; + UpdateManifest = true; + } + + // Handle any state wipe + + if (WipeState) + { + ZEN_WARN("Wiping state at '{}' - reason: '{}'", m_DataRoot, WipeReason); + + std::error_code Ec; + for (const std::filesystem::directory_entry& DirEntry : std::filesystem::directory_iterator{m_DataRoot, Ec}) + { + if (DirEntry.is_directory() && (DirEntry.path().filename() != "logs")) + { + ZEN_INFO("Deleting '{}'", DirEntry.path()); + + DeleteDirectories(DirEntry.path(), Ec); + + if (Ec) + { + ZEN_WARN("Delete of '{}' returned error: '{}'", DirEntry.path(), Ec.message()); + } + } + } + + ZEN_INFO("Wiped all directories in data root"); + + UpdateManifest = true; + } + + // Write manifest + + { + CbObjectWriter Cbo; + Cbo << "schema_version" << ZEN_CFG_SCHEMA_VERSION << "created" << CreatedWhen << "updated" << Now << "state_id" << StateId; + + m_RootManifest = Cbo.Save(); + + if (UpdateManifest) + { + TemporaryFile::SafeWriteFile(ManifestPath, m_RootManifest.GetBuffer().GetView()); + } + + if (!ServerOptions.IsTest) + { + try + { + EmitCentralManifest(ServerOptions.SystemRootDir, StateId, m_RootManifest, ManifestPath); + } + catch (const std::exception& Ex) + { + ZEN_WARN("Unable to emit central manifest: ", Ex.what()); + } + } + } + + // Write state marker + + { + std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; + static const std::string_view StateMarkerContent = "deleting this file will cause " ZEN_APP_NAME " to exit"sv; + WriteFile(StateMarkerPath, IoBuffer(IoBuffer::Wrap, StateMarkerContent.data(), StateMarkerContent.size())); + + EnqueueStateMarkerTimer(); + } + + EnqueueStateExitFlagTimer(); +} + +void +ZenStorageServer::InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions) +{ + ZEN_TRACE_CPU("ZenStorageServer::InitializeStructuredCache"); + + using namespace std::literals; + + ZEN_INFO("instantiating structured cache service"); + ZenCacheStore::Configuration Config; + Config.AllowAutomaticCreationOfNamespaces = true; + Config.Logging = {.EnableWriteLog = ServerOptions.StructuredCacheConfig.WriteLogEnabled, + .EnableAccessLog = ServerOptions.StructuredCacheConfig.AccessLogEnabled}; + + for (const auto& It : ServerOptions.StructuredCacheConfig.PerBucketConfigs) + { + const std::string& BucketName = It.first; + const ZenStructuredCacheBucketConfig& ZenBucketConfig = It.second; + ZenCacheDiskLayer::BucketConfiguration BucketConfig = {.MaxBlockSize = ZenBucketConfig.MaxBlockSize, + .PayloadAlignment = ZenBucketConfig.PayloadAlignment, + .MemCacheSizeThreshold = ZenBucketConfig.MemCacheSizeThreshold, + .LargeObjectThreshold = ZenBucketConfig.LargeObjectThreshold, + .LimitOverwrites = ZenBucketConfig.LimitOverwrites}; + Config.NamespaceConfig.DiskLayerConfig.BucketConfigMap.insert_or_assign(BucketName, BucketConfig); + } + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MaxBlockSize = ServerOptions.StructuredCacheConfig.BucketConfig.MaxBlockSize, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.PayloadAlignment = + ServerOptions.StructuredCacheConfig.BucketConfig.PayloadAlignment, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.MemCacheSizeThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.MemCacheSizeThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = + ServerOptions.StructuredCacheConfig.BucketConfig.LargeObjectThreshold, + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LimitOverwrites = ServerOptions.StructuredCacheConfig.BucketConfig.LimitOverwrites; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTargetFootprintBytes = ServerOptions.StructuredCacheConfig.MemTargetFootprintBytes; + Config.NamespaceConfig.DiskLayerConfig.MemCacheTrimIntervalSeconds = ServerOptions.StructuredCacheConfig.MemTrimIntervalSeconds; + Config.NamespaceConfig.DiskLayerConfig.MemCacheMaxAgeSeconds = ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds; + + if (ServerOptions.IsDedicated) + { + Config.NamespaceConfig.DiskLayerConfig.BucketConfig.LargeObjectThreshold = 128 * 1024 * 1024; + } + + m_CacheStore = new ZenCacheStore(m_GcManager, *m_JobQueue, m_DataRoot / "cache", Config, m_GcManager.GetDiskWriteBlocker()); + m_OpenProcessCache = std::make_unique<OpenProcessCache>(); + + const ZenUpstreamCacheConfig& UpstreamConfig = ServerOptions.UpstreamCacheConfig; + + UpstreamCacheOptions UpstreamOptions; + UpstreamOptions.ReadUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Read)) != 0; + UpstreamOptions.WriteUpstream = (uint8_t(ServerOptions.UpstreamCacheConfig.CachePolicy) & uint8_t(UpstreamCachePolicy::Write)) != 0; + + if (UpstreamConfig.UpstreamThreadCount < 32) + { + UpstreamOptions.ThreadCount = static_cast<uint32_t>(UpstreamConfig.UpstreamThreadCount); + } + + m_UpstreamCache = CreateUpstreamCache(UpstreamOptions, *m_CacheStore, *m_CidStore); + m_UpstreamService = std::make_unique<HttpUpstreamService>(*m_UpstreamCache, *m_AuthMgr); + m_UpstreamCache->Initialize(); + + if (ServerOptions.UpstreamCacheConfig.CachePolicy != UpstreamCachePolicy::Disabled) + { + // Zen upstream + { + std::vector<std::string> ZenUrls = UpstreamConfig.ZenConfig.Urls; + if (!UpstreamConfig.ZenConfig.Dns.empty()) + { + for (const std::string& Dns : UpstreamConfig.ZenConfig.Dns) + { + if (!Dns.empty()) + { + const asio::error_code Err = utils::ResolveHostname(m_IoContext, Dns, "8558"sv, ZenUrls); + if (Err) + { + ZEN_ERROR("resolve of '{}' FAILED, reason '{}'", Dns, Err.message()); + } + } + } + } + + std::erase_if(ZenUrls, [](const auto& Url) { return Url.empty(); }); + + if (!ZenUrls.empty()) + { + const auto ZenEndpointName = UpstreamConfig.ZenConfig.Name.empty() ? "Zen"sv : UpstreamConfig.ZenConfig.Name; + + std::unique_ptr<UpstreamEndpoint> ZenEndpoint = UpstreamEndpoint::CreateZenEndpoint( + {.Name = ZenEndpointName, + .Urls = ZenUrls, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}); + + m_UpstreamCache->RegisterEndpoint(std::move(ZenEndpoint)); + } + } + + // Jupiter upstream + if (UpstreamConfig.JupiterConfig.Url.empty() == false) + { + std::string_view EndpointName = UpstreamConfig.JupiterConfig.Name.empty() ? "Jupiter"sv : UpstreamConfig.JupiterConfig.Name; + + auto Options = JupiterClientOptions{.Name = EndpointName, + .ServiceUrl = UpstreamConfig.JupiterConfig.Url, + .DdcNamespace = UpstreamConfig.JupiterConfig.DdcNamespace, + .BlobStoreNamespace = UpstreamConfig.JupiterConfig.Namespace, + .ConnectTimeout = std::chrono::milliseconds(UpstreamConfig.ConnectTimeoutMilliseconds), + .Timeout = std::chrono::milliseconds(UpstreamConfig.TimeoutMilliseconds)}; + + auto AuthConfig = UpstreamAuthConfig{.OAuthUrl = UpstreamConfig.JupiterConfig.OAuthUrl, + .OAuthClientId = UpstreamConfig.JupiterConfig.OAuthClientId, + .OAuthClientSecret = UpstreamConfig.JupiterConfig.OAuthClientSecret, + .OpenIdProvider = UpstreamConfig.JupiterConfig.OpenIdProvider, + .AccessToken = UpstreamConfig.JupiterConfig.AccessToken}; + + std::unique_ptr<UpstreamEndpoint> JupiterEndpoint = UpstreamEndpoint::CreateJupiterEndpoint(Options, AuthConfig, *m_AuthMgr); + + m_UpstreamCache->RegisterEndpoint(std::move(JupiterEndpoint)); + } + } + + m_StructuredCacheService = std::make_unique<HttpStructuredCacheService>(*m_CacheStore, + *m_CidStore, + m_StatsService, + m_StatusService, + *m_UpstreamCache, + m_GcManager.GetDiskWriteBlocker(), + *m_OpenProcessCache); + + m_StatsReporter.AddProvider(m_CacheStore.Get()); + m_StatsReporter.AddProvider(m_CidStore.get()); + m_StatsReporter.AddProvider(m_BuildCidStore.get()); +} + +void +ZenStorageServer::Run() +{ + if (m_ProcessMonitor.IsActive()) + { + CheckOwnerPid(); + } + + if (!m_TestMode) + { + ZEN_INFO( + "__________ _________ __ \n" + "\\____ /____ ____ / _____// |_ ___________ ____ \n" + " / // __ \\ / \\ \\_____ \\\\ __\\/ _ \\_ __ \\_/ __ \\ \n" + " / /\\ ___/| | \\ / \\| | ( <_> ) | \\/\\ ___/ \n" + "/_______ \\___ >___| / /_______ /|__| \\____/|__| \\___ >\n" + " \\/ \\/ \\/ \\/ \\/ \n"); + } + + ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); + +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); + } +#endif + +#if ZEN_USE_SENTRY + ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); + if (m_UseSentry) + { + SentryIntegration::ClearCaches(); + } +#endif + + if (m_DebugOptionForcedCrash) + { + ZEN_DEBUG_BREAK(); + } + + const bool IsInteractiveMode = IsInteractiveSession() && !m_TestMode; + + SetNewState(kRunning); + + OnReady(); + + if (!m_StartupScrubOptions.empty()) + { + using namespace std::literals; + + ZEN_INFO("triggering scrub with settings: '{}'", m_StartupScrubOptions); + + bool DoScrub = true; + bool DoWait = false; + GcScheduler::TriggerScrubParams ScrubParams; + + ForEachStrTok(m_StartupScrubOptions, ',', [&](std::string_view Token) { + if (Token == "nocas"sv) + { + ScrubParams.SkipCas = true; + } + else if (Token == "nodelete"sv) + { + ScrubParams.SkipDelete = true; + } + else if (Token == "nogc"sv) + { + ScrubParams.SkipGc = true; + } + else if (Token == "no"sv) + { + DoScrub = false; + } + else if (Token == "wait"sv) + { + DoWait = true; + } + return true; + }); + + if (DoScrub) + { + m_GcScheduler.TriggerScrub(ScrubParams); + + if (DoWait) + { + auto State = m_GcScheduler.Status(); + + while ((State != GcSchedulerStatus::kRunning) && (State != GcSchedulerStatus::kStopped)) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("waiting for Scrub/GC to complete..."); + + while (State == GcSchedulerStatus::kRunning) + { + Sleep(500); + + State = m_GcScheduler.Status(); + } + + ZEN_INFO("Scrub/GC completed"); + } + } + } + + if (m_IsPowerCycle) + { + ZEN_INFO("Power cycle mode enabled -- shutting down"); + RequestExit(0); + } + + m_Http->Run(IsInteractiveMode); + + SetNewState(kShuttingDown); + + ZEN_INFO(ZEN_APP_NAME " exiting"); +} + +void +ZenStorageServer::Cleanup() +{ + ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_INFO(ZEN_APP_NAME " cleaning up"); + try + { + m_IoContext.stop(); + if (m_IoRunner.joinable()) + { + m_IoRunner.join(); + } + + if (m_Http) + { + m_Http->Close(); + } + + if (m_JobQueue) + { + m_JobQueue->Stop(); + } + + m_StatsReporter.Shutdown(); + m_GcScheduler.Shutdown(); + + Flush(); + + m_AdminService.reset(); + m_VfsService.reset(); + m_VfsServiceImpl.reset(); + m_ObjStoreService.reset(); + m_FrontendService.reset(); + + m_BuildStoreService.reset(); + m_BuildStore = {}; + m_BuildCidStore.reset(); + + m_StructuredCacheService.reset(); + m_UpstreamService.reset(); + m_UpstreamCache.reset(); + m_CacheStore = {}; + m_OpenProcessCache.reset(); + + m_HttpWorkspacesService.reset(); + m_Workspaces.reset(); + m_HttpProjectService.reset(); + m_ProjectStore = {}; + m_CidStore.reset(); + m_AuthService.reset(); + m_AuthMgr.reset(); + m_Http = {}; + + ShutdownWorkerPools(); + + m_JobQueue.reset(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); + } +} + +void +ZenStorageServer::EnqueueStateMarkerTimer() +{ + ZEN_MEMSCOPE(GetZenserverTag()); + m_StateMarkerTimer.expires_after(std::chrono::seconds(5)); + m_StateMarkerTimer.async_wait([this](const asio::error_code&) { CheckStateMarker(); }); + EnsureIoRunner(); +} + +void +ZenStorageServer::CheckStateMarker() +{ + ZEN_MEMSCOPE(GetZenserverTag()); + std::filesystem::path StateMarkerPath = m_DataRoot / "state_marker"; + try + { + if (!IsFile(StateMarkerPath)) + { + ZEN_WARN("state marker at {} has been deleted, exiting", StateMarkerPath); + RequestExit(1); + return; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("state marker at {} could not be checked, reason: '{}'", StateMarkerPath, Ex.what()); + RequestExit(1); + return; + } + EnqueueStateMarkerTimer(); +} + +void +ZenStorageServer::Flush() +{ + ZEN_TRACE_CPU("ZenStorageServer::Flush"); + + if (m_CidStore) + m_CidStore->Flush(); + + if (m_StructuredCacheService) + m_StructuredCacheService->Flush(); + + if (m_ProjectStore) + m_ProjectStore->Flush(); + + if (m_BuildCidStore) + m_BuildCidStore->Flush(); +} + +////////////////////////////////////////////////////////////////////////// + +ZenStorageServerMain::ZenStorageServerMain(ZenStorageServerOptions& ServerOptions) +: ZenServerMain(ServerOptions) +, m_ServerOptions(ServerOptions) +{ +} + +void +ZenStorageServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) +{ + ZenStorageServer Server; + Server.SetDataRoot(m_ServerOptions.DataDir); + Server.SetContentRoot(m_ServerOptions.ContentDir); + Server.SetTestMode(m_ServerOptions.IsTest); + Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + + auto ServerCleanup = MakeGuard([&Server] { Server.Cleanup(); }); + + int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); + if (EffectiveBasePort == -1) + { + // Server.Initialize has already logged what the issue is - just exit with failure code here. + std::exit(1); + } + + Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); + if (EffectiveBasePort != m_ServerOptions.BasePort) + { + ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort); + m_ServerOptions.BasePort = EffectiveBasePort; + } + + std::unique_ptr<std::thread> ShutdownThread; + std::unique_ptr<NamedEvent> ShutdownEvent; + + ExtendableStringBuilder<64> ShutdownEventName; + ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown"; + ShutdownEvent.reset(new NamedEvent{ShutdownEventName}); + + // Monitor shutdown signals + + ShutdownThread.reset(new std::thread{[&] { + SetCurrentThreadName("shutdown_monitor"); + + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId()); + + if (ShutdownEvent->Wait()) + { + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); + Server.RequestExit(0); + } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } + }}); + + auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { + ReportServiceStatus(ServiceStatus::Stopping); + + if (ShutdownEvent) + { + ShutdownEvent->Set(); + } + if (ShutdownThread && ShutdownThread->joinable()) + { + ShutdownThread->join(); + } + }); + + // If we have a parent process, establish the mechanisms we need + // to be able to communicate readiness with the parent + + Server.SetIsReadyFunc([&] { + std::error_code Ec; + m_LockFile.Update(MakeLockData(true), Ec); + ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); + }); + + Server.Run(); +} + +} // namespace zen diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h new file mode 100644 index 000000000..e4c31399d --- /dev/null +++ b/src/zenserver/storage/zenstorageserver.h @@ -0,0 +1,113 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenserver.h" + +#include <zenhttp/auth/authmgr.h> +#include <zenhttp/auth/authservice.h> +#include <zenhttp/httptest.h> +#include <zenstore/cache/structuredcachestore.h> +#include <zenstore/gc.h> +#include <zenstore/projectstore.h> + +#include "admin/admin.h" +#include "buildstore/httpbuildstore.h" +#include "cache/httpstructuredcache.h" +#include "diag/diagsvcs.h" +#include "frontend/frontend.h" +#include "objectstore/objectstore.h" +#include "projectstore/httpprojectstore.h" +#include "stats/statsreporter.h" +#include "upstream/upstream.h" +#include "vfs/vfsservice.h" +#include "workspaces/httpworkspaces.h" + +namespace zen { + +class ZenStorageServer : public ZenServerBase +{ + ZenStorageServer& operator=(ZenStorageServer&&) = delete; + ZenStorageServer(ZenStorageServer&&) = delete; + +public: + ZenStorageServer(); + ~ZenStorageServer(); + + void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetTestMode(bool State) { m_TestMode = State; } + void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } + void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } + + int Initialize(const ZenStorageServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); + void Run(); + void Cleanup(); + +private: + void InitializeState(const ZenStorageServerOptions& ServerOptions); + void InitializeStructuredCache(const ZenStorageServerOptions& ServerOptions); + void Flush(); + + bool m_IsDedicatedMode = false; + bool m_TestMode = false; + bool m_DebugOptionForcedCrash = false; + std::string m_StartupScrubOptions; + CbObject m_RootManifest; + std::filesystem::path m_DataRoot; + std::filesystem::path m_ContentRoot; + asio::steady_timer m_StateMarkerTimer{m_IoContext}; + + void EnqueueStateMarkerTimer(); + void CheckStateMarker(); + + std::unique_ptr<AuthMgr> m_AuthMgr; + std::unique_ptr<HttpAuthService> m_AuthService; + void InitializeAuthentication(const ZenStorageServerOptions& ServerOptions); + + void InitializeServices(const ZenStorageServerOptions& ServerOptions); + void RegisterServices(); + + HttpStatsService m_StatsService; + std::unique_ptr<JobQueue> m_JobQueue; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr<CidStore> m_CidStore; + Ref<ZenCacheStore> m_CacheStore; + std::unique_ptr<OpenProcessCache> m_OpenProcessCache; + HttpTestService m_TestService; + std::unique_ptr<CidStore> m_BuildCidStore; + std::unique_ptr<BuildStore> m_BuildStore; + +#if ZEN_WITH_TESTS + HttpTestingService m_TestingService; +#endif + + RefPtr<ProjectStore> m_ProjectStore; + std::unique_ptr<VfsServiceImpl> m_VfsServiceImpl; + std::unique_ptr<HttpProjectService> m_HttpProjectService; + std::unique_ptr<Workspaces> m_Workspaces; + std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService; + std::unique_ptr<UpstreamCache> m_UpstreamCache; + std::unique_ptr<HttpUpstreamService> m_UpstreamService; + std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService; + std::unique_ptr<HttpFrontendService> m_FrontendService; + std::unique_ptr<HttpObjectStoreService> m_ObjStoreService; + std::unique_ptr<HttpBuildStoreService> m_BuildStoreService; + std::unique_ptr<VfsService> m_VfsService; + std::unique_ptr<HttpAdminService> m_AdminService; +}; + +class ZenStorageServerMain : public ZenServerMain +{ +public: + ZenStorageServerMain(ZenStorageServerOptions& ServerOptions); + virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; + + ZenStorageServerMain(const ZenStorageServerMain&) = delete; + ZenStorageServerMain& operator=(const ZenStorageServerMain&) = delete; + +private: + ZenStorageServerOptions& m_ServerOptions; +}; + +} // namespace zen |