// Copyright Epic Games, Inc. All Rights Reserved. #include "admin.h" #include #include #include #include #include #include #if ZEN_WITH_TRACE # include #endif // ZEN_WITH_TRACE #if ZEN_USE_MIMALLOC # include #endif #include #include #include #include "config.h" #include "projectstore/projectstore.h" #include namespace zen { struct DirStats { uint64_t FileCount = 0; uint64_t DirCount = 0; uint64_t ByteCount = 0; }; DirStats GetStatsForDirectory(std::filesystem::path Dir) { if (!std::filesystem::exists(Dir)) return {}; FileSystemTraversal Traversal; struct StatsTraversal : public FileSystemTraversal::TreeVisitor { virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override { ZEN_UNUSED(Parent, File); ++TotalFileCount; TotalBytes += FileSize; } virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { ++TotalDirCount; return true; } uint64_t TotalBytes = 0; uint64_t TotalFileCount = 0; uint64_t TotalDirCount = 0; DirStats GetStats() { return {.FileCount = TotalFileCount, .DirCount = TotalDirCount, .ByteCount = TotalBytes}; } }; StatsTraversal DirTraverser; Traversal.TraverseFileSystem(Dir, DirTraverser); return DirTraverser.GetStats(); } struct StateDiskStats { DirStats CacheStats; DirStats CasStats; DirStats ProjectStats; }; StateDiskStats GetStatsForStateDirectory(std::filesystem::path StateDir) { StateDiskStats Stats; Stats.CacheStats = GetStatsForDirectory(StateDir / "cache"); Stats.CasStats = GetStatsForDirectory(StateDir / "cas"); Stats.ProjectStats = GetStatsForDirectory(StateDir / "projects"); return Stats; } HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, const LogPaths& LogPaths, const ZenServerOptions& ServerOptions) : m_GcScheduler(Scheduler) , m_BackgroundJobQueue(BackgroundJobQueue) , m_CacheStore(CacheStore) , m_CidStore(CidStore) , m_ProjectStore(ProjectStore) , m_LogPaths(LogPaths) , m_ServerOptions(ServerOptions) { using namespace std::literals; m_Router.RegisterRoute( "health", [](HttpRouterRequest& Req) { CbObjectWriter Obj; Obj.AddBool("ok", true); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.AddPattern("jobid", "([[:digit:]]+?)"); m_Router.RegisterRoute( "jobs", [&](HttpRouterRequest& Req) { std::vector Jobs = m_BackgroundJobQueue.GetJobs(); CbObjectWriter Obj; Obj.BeginArray("jobs"); for (const auto& Job : Jobs) { Obj.BeginObject(); Obj.AddInteger("Id", Job.Id.Id); Obj.AddString("Status", JobQueue::ToString(Job.Status)); Obj.EndObject(); } Obj.EndArray(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "jobs/{jobid}", [&](HttpRouterRequest& Req) { const auto& JobIdString = Req.GetCapture(1); std::optional JobIdArg = ParseInt(JobIdString); if (!JobIdArg) { Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } JobId Id{.Id = JobIdArg.value_or(0)}; if (Id.Id == 0) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, ZenContentType::kText, fmt::format("Invalid Job Id: {}", Id.Id)); } std::optional CurrentState = m_BackgroundJobQueue.Get(Id); if (!CurrentState) { return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { if (!State.CurrentOp.empty()) { Obj.AddString("CurrentOp"sv, State.CurrentOp); Obj.AddInteger("CurrentOpPercentComplete"sv, State.CurrentOpPercentComplete); } if (!State.Messages.empty()) { Obj.BeginArray("Messages"); for (const std::string& Message : State.Messages) { Obj.AddString(Message); } Obj.EndArray(); } if (!State.AbortReason.empty()) { Obj.AddString("AbortReason"sv, State.AbortReason); } }; auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { auto Age = End - Start; auto Milliseconds = std::chrono::duration_cast(Age); return Milliseconds.count() / 1000.0; }; const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); switch (CurrentState->Status) { case JobQueue::Status::Queued: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Queued"sv); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, Now)); Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Running: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Running"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, Now)); Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Aborted: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Aborted"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Completed: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Complete"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; } }, HttpVerb::kGet); m_Router.RegisterRoute( "jobs/{jobid}", [&](HttpRouterRequest& Req) { const auto& JobIdString = Req.GetCapture(1); std::optional JobIdArg = ParseInt(JobIdString); if (!JobIdArg) { Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } JobId Id{.Id = JobIdArg.value_or(0)}; if (m_BackgroundJobQueue.CancelJob(Id)) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } else { Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } }, HttpVerb::kDelete); m_Router.RegisterRoute( "gc", [this](HttpRouterRequest& Req) { const GcSchedulerState State = m_GcScheduler.GetState(); const HttpServerRequest::QueryParams Params = Req.ServerRequest().GetQueryParams(); bool Details = false; if (auto Param = Params.GetValue("details"); Param == "true") { Details = true; } CbObjectWriter Response; Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv); Response.BeginObject("Config"); { Response << "RootDirectory" << State.Config.RootDirectory.string(); Response << "MonitorInterval" << ToTimeSpan(State.Config.MonitorInterval); Response << "Interval" << ToTimeSpan(State.Config.Interval); Response << "MaxCacheDuration" << ToTimeSpan(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << ToTimeSpan(State.Config.MaxProjectStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); Response << "LightweightInterval" << ToTimeSpan(State.Config.LightweightInterval); Response << "UseGCVersion" << ((State.Config.UseGCVersion == GcVersion::kV1) ? "1" : "2"); Response << "CompactBlockUsageThresholdPercent" << State.Config.CompactBlockUsageThresholdPercent; Response << "Verbose" << State.Config.Verbose; } Response.EndObject(); Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; Response << "HasDiskReserve" << State.HasDiskReserve; Response << "DiskSize" << NiceBytes(State.DiskSize); Response << "DiskUsed" << NiceBytes(State.DiskUsed); Response << "DiskFree" << NiceBytes(State.DiskFree); Response.BeginObject("FullGC"); { Response << "LastTime" << ToDateTime(State.LastFullGcTime); Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilFullGc); if (State.Config.DiskSizeSoftLimit != 0) { Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); } if (State.LastFullGCV2Result) { const bool HumanReadable = true; WriteGCResult(Response, State.LastFullGCV2Result.value(), HumanReadable, Details); } else { Response << "LastDuration" << ToTimeSpan(State.LastFullGcDuration); Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); } } Response.EndObject(); Response.BeginObject("LightweightGC"); { Response << "LastTime" << ToDateTime(State.LastLightweightGcTime); Response << "TimeToNext" << ToTimeSpan(State.RemainingTimeUntilLightweightGc); if (State.LastLightweightGCV2Result) { const bool HumanReadable = true; WriteGCResult(Response, State.LastLightweightGCV2Result.value(), HumanReadable, Details); } else { Response << "LastDuration" << ToTimeSpan(State.LastLightweightGcDuration); Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); } } Response.EndObject(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "gc", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); GcScheduler::TriggerGcParams GcParams; if (auto Param = Params.GetValue("smallobjects"); Param.empty() == false) { GcParams.CollectSmallObjects = Param == "true"sv; } if (auto Param = Params.GetValue("maxcacheduration"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.MaxCacheDuration = std::chrono::seconds(Value.value()); } } if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value()); } } if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.DiskSizeSoftLimit = Value.value(); } } if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) { GcParams.SkipCid = Param == "true"sv; } if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) { GcParams.SkipDelete = Param == "true"sv; } if (auto Param = Params.GetValue("forceusegcv1"); Param.empty() == false) { GcParams.ForceGCVersion = GcVersion::kV1; } if (auto Param = Params.GetValue("forceusegcv2"); Param.empty() == false) { GcParams.ForceGCVersion = GcVersion::kV2; } if (auto Param = Params.GetValue("compactblockthreshold"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.CompactBlockUsageThresholdPercent = Value.value(); } } if (auto Param = Params.GetValue("verbose"); Param.empty() == false) { GcParams.Verbose = Param == "true"sv; } const bool Started = m_GcScheduler.TriggerGc(GcParams); CbObjectWriter Response; Response << "Status"sv << (Started ? "Started"sv : "Running"sv); HttpReq.WriteResponse(HttpResponseCode::Accepted, Response.Save()); }, HttpVerb::kPost); m_Router.RegisterRoute( "gc-stop", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); if (m_GcScheduler.CancelGC()) { return HttpReq.WriteResponse(HttpResponseCode::Accepted); } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); #if ZEN_USE_MIMALLOC m_Router.RegisterRoute( "mi_collect", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); bool Force = false; if (auto Param = Params.GetValue("force"); Param.empty() == false) { Force = (Param == "true"sv); } ExtendableStringBuilder<256> MiStats; ExtendableStringBuilder<256> MiStatsAfter; auto MiOutputFun = [](const char* msg, void* arg) { StringBuilderBase* StarsSb = reinterpret_cast(arg); StarsSb->AppendAscii(msg); }; mi_stats_print_out(MiOutputFun, static_cast(&MiStats)); mi_collect(Force); mi_stats_print_out(MiOutputFun, static_cast(&MiStatsAfter)); CbObjectWriter Response; Response << "force"sv << Force; Response << "stats_before"sv << MiStats; Response << "stats_after"sv << MiStatsAfter; HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kPost); #endif m_Router.RegisterRoute( "scrub", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); GcScheduler::TriggerScrubParams ScrubParams; ScrubParams.MaxTimeslice = std::chrono::seconds(100); if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) { ScrubParams.SkipDelete = (Param == "true"sv); } if (auto Param = Params.GetValue("skipgc"); Param.empty() == false) { ScrubParams.SkipGc = (Param == "true"sv); } if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) { ScrubParams.SkipCas = (Param == "true"sv); } m_GcScheduler.TriggerScrub(ScrubParams); CbObjectWriter Response; Response << "ok"sv << true; Response << "skip_delete" << ScrubParams.SkipDelete; Response << "skip_gc" << ScrubParams.SkipGc; Response << "skip_cas" << ScrubParams.SkipCas; Response << "max_time" << TimeSpan(0, 0, gsl::narrow(ScrubParams.MaxTimeslice.count())); HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kPost); m_Router.RegisterRoute( "", [](HttpRouterRequest& Req) { CbObject Payload = Req.ServerRequest().ReadPayloadObject(); CbObjectWriter Obj; Obj.AddBool("ok", true); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kPost); #if ZEN_WITH_TRACE m_Router.RegisterRoute( "trace", [this](HttpRouterRequest& Req) { bool Enabled = IsTracing(); return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); }, HttpVerb::kGet); m_Router.RegisterRoute( "trace/start", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); TraceType Type = TraceType::None; std::string HostOrPath; if (auto Param = Params.GetValue("file"); Param.empty() == false) { Type = TraceType::File; HostOrPath = Param; } if (auto Param = Params.GetValue("host"); Param.empty() == false) { Type = TraceType::Network; HostOrPath = Param; } if (Type == TraceType::None) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid trace type, use `file` or `host`"sv); } if (IsTracing()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is already enabled"sv); } TraceStart("zenserver", HostOrPath.c_str(), Type); return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); }, HttpVerb::kPost); m_Router.RegisterRoute( "trace/stop", [this](HttpRouterRequest& Req) { if (!IsTracing()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); } if (TraceStop()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); } else { return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, "Failed stopping trace"); } }, HttpVerb::kPost); #endif // ZEN_WITH_TRACE m_Router.RegisterRoute( "info", [this](HttpRouterRequest& Req) { CbObjectWriter Obj; Obj << "root" << m_ServerOptions.SystemRootDir.generic_wstring(); Obj << "install" << (m_ServerOptions.SystemRootDir / "Install").generic_wstring(); Obj.BeginObject("primary"); Obj << "data" << m_ServerOptions.DataDir.generic_wstring(); try { auto Stats = GetStatsForStateDirectory(m_ServerOptions.DataDir); auto EmitStats = [&](std::string_view Tag, const DirStats& Stats) { Obj.BeginObject(Tag); Obj << "bytes" << Stats.ByteCount; Obj << "files" << Stats.FileCount; Obj << "dirs" << Stats.DirCount; Obj.EndObject(); }; EmitStats("cache", Stats.CacheStats); EmitStats("cas", Stats.CasStats); EmitStats("project", Stats.ProjectStats); } catch (const std::exception& Ex) { ZEN_WARN("exception in disk stats gathering for '{}': {}", m_ServerOptions.DataDir, Ex.what()); } Obj.EndObject(); try { std::vector Manifests = ReadAllCentralManifests(m_ServerOptions.SystemRootDir); Obj.BeginArray("known"); for (const auto& Manifest : Manifests) { Obj.AddObject(Manifest); } Obj.EndArray(); } catch (const std::exception& Ex) { ZEN_WARN("exception in state gathering for '{}': {}", m_ServerOptions.SystemRootDir, Ex.what()); } Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "logs", [this](HttpRouterRequest& Req) { CbObjectWriter Obj; auto LogLevel = logging::level::ToStringView(logging::GetLogLevel()); Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); Obj.BeginObject("cache"); if (m_CacheStore) { const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration(); Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); } Obj.EndObject(); Obj.BeginObject("http"); { Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); } Obj.EndObject(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "logs", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); bool SetCacheLogConfig = false; ExtendableStringBuilder<256> StringBuilder; if (m_CacheStore) { ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging; if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) { LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; SetCacheLogConfig = true; } if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) { LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; SetCacheLogConfig = true; } if (SetCacheLogConfig) { m_CacheStore->SetLoggingConfig(LoggingConfig); StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", LoggingConfig.EnableWriteLog ? "true" : "false", LoggingConfig.EnableAccessLog ? "true" : "false")); } } if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) { logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param); std::string_view LogLevel = logging::level::ToStringView(NewLevel); if (LogLevel != Param) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, fmt::format("Invalid log level '{}'", Param)); } logging::SetLogLevel(NewLevel); if (StringBuilder.Size() > 0) { StringBuilder.Append(", "); } StringBuilder.Append("loglevel: "); StringBuilder.Append(Param); } return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); }, HttpVerb::kPost); m_Router.RegisterRoute( "flush", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); if (m_CidStore) { m_CidStore->Flush(); } if (m_CacheStore) { m_CacheStore->Flush(); } if (m_ProjectStore) { m_ProjectStore->Flush(); } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); } HttpAdminService::~HttpAdminService() { } const char* HttpAdminService::BaseUri() const { return "/admin/"; } void HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) { m_Router.HandleRequest(Request); } } // namespace zen