// Copyright Epic Games, Inc. All Rights Reserved. #include "admin.h" #include #include #include #include #if ZEN_WITH_TRACE # include #endif // ZEN_WITH_TRACE #if ZEN_USE_MIMALLOC # include #endif #include #include #include "cache/structuredcachestore.h" #include "projectstore/projectstore.h" #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END namespace zen { HttpAdminService::HttpAdminService(GcScheduler& Scheduler, JobQueue& BackgroundJobQueue, ZenCacheStore* CacheStore, CidStore* CidStore, ProjectStore* ProjectStore, const LogPaths& LogPaths) : m_GcScheduler(Scheduler) , m_BackgroundJobQueue(BackgroundJobQueue) , m_CacheStore(CacheStore) , m_CidStore(CidStore) , m_ProjectStore(ProjectStore) , m_LogPaths(LogPaths) { using namespace std::literals; m_Router.RegisterRoute( "health", [](HttpRouterRequest& Req) { CbObjectWriter Obj; Obj.AddBool("ok", true); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.AddPattern("jobid", "([[:digit:]]+?)"); m_Router.RegisterRoute( "jobs", [&](HttpRouterRequest& Req) { std::vector Jobs = m_BackgroundJobQueue.GetJobs(); CbObjectWriter Obj; Obj.BeginArray("jobs"); for (const auto& Job : Jobs) { Obj.BeginObject(); Obj.AddInteger("Id", Job.Id.Id); Obj.AddString("Status", JobQueue::ToString(Job.Status)); Obj.EndObject(); } Obj.EndArray(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "jobs/{jobid}", [&](HttpRouterRequest& Req) { const auto& JobIdString = Req.GetCapture(1); std::optional JobIdArg = ParseInt(JobIdString); if (!JobIdArg) { Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } JobId Id{.Id = JobIdArg.value_or(0)}; if (Id.Id == 0) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, ZenContentType::kText, fmt::format("Invalid Job Id: {}", Id.Id)); } std::optional CurrentState = m_BackgroundJobQueue.Get(Id); if (!CurrentState) { return Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } auto WriteState = [](CbObjectWriter& Obj, const JobQueue::State& State) { if (!State.CurrentOp.empty()) { Obj.AddString("CurrentOp"sv, State.CurrentOp); Obj.AddInteger("CurrentOpPercentComplete"sv, State.CurrentOpPercentComplete); } if (!State.Messages.empty()) { Obj.BeginArray("Messages"); for (const std::string& Message : State.Messages) { Obj.AddString(Message); } Obj.EndArray(); } }; auto GetAgeAsSeconds = [](std::chrono::system_clock::time_point Start, std::chrono::system_clock::time_point End) { auto Age = End - Start; auto Milliseconds = std::chrono::duration_cast(Age); return Milliseconds.count() / 1000.0; }; const std::chrono::system_clock::time_point Now = std::chrono::system_clock::now(); switch (CurrentState->Status) { case JobQueue::Status::Queued: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Queued"sv); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, Now)); Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Running: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Running"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, Now)); Obj.AddInteger("WorkerThread", CurrentState->WorkerThreadId); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Aborted: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Aborted"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; case JobQueue::Status::Completed: { CbObjectWriter Obj; Obj.AddString("Name"sv, CurrentState->Name); Obj.AddString("Status"sv, "Complete"sv); WriteState(Obj, CurrentState->State); Obj.AddFloat("QueueTimeS", GetAgeAsSeconds(CurrentState->CreateTime, CurrentState->StartTime)); Obj.AddFloat("RunTimeS", GetAgeAsSeconds(CurrentState->StartTime, CurrentState->EndTime)); Obj.AddFloat("CompleteTimeS", GetAgeAsSeconds(CurrentState->EndTime, Now)); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); } break; } }, HttpVerb::kGet); m_Router.RegisterRoute( "jobs/{jobid}", [&](HttpRouterRequest& Req) { const auto& JobIdString = Req.GetCapture(1); std::optional JobIdArg = ParseInt(JobIdString); if (!JobIdArg) { Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest); } JobId Id{.Id = JobIdArg.value_or(0)}; if (m_BackgroundJobQueue.CancelJob(Id)) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } else { Req.ServerRequest().WriteResponse(HttpResponseCode::NotFound); } }, HttpVerb::kDelete); m_Router.RegisterRoute( "gc", [this](HttpRouterRequest& Req) { const GcSchedulerState State = m_GcScheduler.GetState(); auto SecondsToString = [](std::chrono::seconds Secs) { return NiceTimeSpanMs(uint64_t(std::chrono::milliseconds(Secs).count())); }; CbObjectWriter Response; Response << "Status"sv << (GcSchedulerStatus::kIdle == State.Status ? "Idle"sv : "Running"sv); Response.BeginObject("Config"); { Response << "RootDirectory" << State.Config.RootDirectory.string(); Response << "MonitorInterval" << SecondsToString(State.Config.MonitorInterval); Response << "Interval" << SecondsToString(State.Config.Interval); Response << "MaxCacheDuration" << SecondsToString(State.Config.MaxCacheDuration); Response << "MaxProjectStoreDuration" << SecondsToString(State.Config.MaxProjectStoreDuration); Response << "CollectSmallObjects" << State.Config.CollectSmallObjects; Response << "Enabled" << State.Config.Enabled; Response << "DiskReserveSize" << NiceBytes(State.Config.DiskReserveSize); Response << "DiskSizeSoftLimit" << NiceBytes(State.Config.DiskSizeSoftLimit); Response << "MinimumFreeDiskSpaceToAllowWrites" << NiceBytes(State.Config.MinimumFreeDiskSpaceToAllowWrites); Response << "LightweightInterval" << SecondsToString(State.Config.LightweightInterval); } Response.EndObject(); Response << "AreDiskWritesBlocked" << State.AreDiskWritesBlocked; Response << "HasDiskReserve" << State.HasDiskReserve; Response << "DiskSize" << NiceBytes(State.DiskSize); Response << "DiskUsed" << NiceBytes(State.DiskUsed); Response << "DiskFree" << NiceBytes(State.DiskFree); Response.BeginObject("FullGC"); { Response << "LastTime" << fmt::format("{}", State.LastFullGcTime); Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilFullGc); if (State.Config.DiskSizeSoftLimit != 0) { Response << "SpaceToNext" << NiceBytes(State.RemainingSpaceUntilFullGC); } Response << "LastDuration" << NiceTimeSpanMs(State.LastFullGcDuration.count()); Response << "LastDiskFreed" << NiceBytes(State.LastFullGCDiff.DiskSize); Response << "LastMemoryFreed" << NiceBytes(State.LastFullGCDiff.MemorySize); } Response.EndObject(); Response.BeginObject("LightweightGC"); { Response << "LastTime" << fmt::format("{}", State.LastLightweightGcTime); Response << "TimeToNext" << SecondsToString(State.RemainingTimeUntilLightweightGc); Response << "LastDuration" << NiceTimeSpanMs(State.LastLightweightGcDuration.count()); Response << "LastDiskFreed" << NiceBytes(State.LastLightweightGCDiff.DiskSize); Response << "LastMemoryFreed" << NiceBytes(State.LastLightweightGCDiff.MemorySize); } Response.EndObject(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "gc", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); GcScheduler::TriggerGcParams GcParams; if (auto Param = Params.GetValue("smallobjects"); Param.empty() == false) { GcParams.CollectSmallObjects = Param == "true"sv; } if (auto Param = Params.GetValue("maxcacheduration"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.MaxCacheDuration = std::chrono::seconds(Value.value()); } } if (auto Param = Params.GetValue("maxprojectstoreduration"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.MaxProjectStoreDuration = std::chrono::seconds(Value.value()); } } if (auto Param = Params.GetValue("disksizesoftlimit"); Param.empty() == false) { if (auto Value = ParseInt(Param)) { GcParams.DiskSizeSoftLimit = Value.value(); } } if (auto Param = Params.GetValue("skipcid"); Param.empty() == false) { GcParams.SkipCid = Param == "true"sv; } if (auto Param = Params.GetValue("skipdelete"); Param.empty() == false) { GcParams.SkipDelete = Param == "true"sv; } const bool Started = m_GcScheduler.TriggerGc(GcParams); CbObjectWriter Response; Response << "Status"sv << (Started ? "Started"sv : "Running"sv); HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kPost); #if ZEN_USE_MIMALLOC m_Router.RegisterRoute( "mi_collect", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); bool Force = false; if (auto Param = Params.GetValue("force"); Param.empty() == false) { Force = (Param == "true"sv); } ExtendableStringBuilder<256> MiStats; ExtendableStringBuilder<256> MiStatsAfter; auto MiOutputFun = [](const char* msg, void* arg) { StringBuilderBase* StarsSb = reinterpret_cast(arg); StarsSb->AppendAscii(msg); }; mi_stats_print_out(MiOutputFun, static_cast(&MiStats)); mi_collect(Force); mi_stats_print_out(MiOutputFun, static_cast(&MiStatsAfter)); CbObjectWriter Response; Response << "force"sv << Force; Response << "stats_before"sv << MiStats; Response << "stats_after"sv << MiStatsAfter; HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kPost); #endif m_Router.RegisterRoute( "scrub", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); GcScheduler::TriggerScrubParams ScrubParams; ScrubParams.MaxTimeslice = std::chrono::seconds(100); m_GcScheduler.TriggerScrub(ScrubParams); CbObjectWriter Response; Response << "ok"sv << true; HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); }, HttpVerb::kPost); m_Router.RegisterRoute( "", [](HttpRouterRequest& Req) { CbObject Payload = Req.ServerRequest().ReadPayloadObject(); CbObjectWriter Obj; Obj.AddBool("ok", true); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kPost); #if ZEN_WITH_TRACE m_Router.RegisterRoute( "trace", [this](HttpRouterRequest& Req) { bool Enabled = IsTracing(); return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Enabled ? "enabled" : "disabled"); }, HttpVerb::kGet); m_Router.RegisterRoute( "trace/start", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); TraceType Type = TraceType::None; std::string HostOrPath; if (auto Param = Params.GetValue("file"); Param.empty() == false) { Type = TraceType::File; HostOrPath = Param; } if (auto Param = Params.GetValue("host"); Param.empty() == false) { Type = TraceType::Network; HostOrPath = Param; } if (Type == TraceType::None) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid trace type, use `file` or `host`"sv); } if (IsTracing()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is already enabled"sv); } TraceStart(HostOrPath.c_str(), Type); return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing started"); }, HttpVerb::kPost); m_Router.RegisterRoute( "trace/stop", [this](HttpRouterRequest& Req) { if (!IsTracing()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Tracing is not enabled"sv); } if (TraceStop()) { return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "Tracing stopped"); } else { return Req.ServerRequest().WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, "Failed stopping trace"); } }, HttpVerb::kPost); #endif // ZEN_WITH_TRACE m_Router.RegisterRoute( "logs", [this](HttpRouterRequest& Req) { CbObjectWriter Obj; spdlog::string_view_t LogLevel = spdlog::level::to_string_view(spdlog::get_level()); Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); Obj.BeginObject("cache"); if (m_CacheStore) { const ZenCacheStore::Configuration& CacheConfig = m_CacheStore->GetConfiguration(); Obj.AddString("Logfile", PathToUtf8(m_LogPaths.CacheLogPath)); Obj.AddBool("Write", CacheConfig.Logging.EnableWriteLog); Obj.AddBool("Access", CacheConfig.Logging.EnableAccessLog); } Obj.EndObject(); Obj.BeginObject("http"); { Obj.AddString("Logfile", PathToUtf8(m_LogPaths.HttpLogPath)); } Obj.EndObject(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); m_Router.RegisterRoute( "logs", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); const HttpServerRequest::QueryParams Params = HttpReq.GetQueryParams(); bool SetCacheLogConfig = false; ExtendableStringBuilder<256> StringBuilder; if (m_CacheStore) { ZenCacheStore::Configuration::LogConfig LoggingConfig = m_CacheStore->GetConfiguration().Logging; if (std::string Param(Params.GetValue("cacheenablewritelog")); Param.empty() == false) { LoggingConfig.EnableWriteLog = StrCaseCompare(Param.c_str(), "true") == 0; SetCacheLogConfig = true; } if (std::string Param(Params.GetValue("cacheenableaccesslog")); Param.empty() == false) { LoggingConfig.EnableAccessLog = StrCaseCompare(Param.c_str(), "true") == 0; SetCacheLogConfig = true; } if (SetCacheLogConfig) { m_CacheStore->SetLoggingConfig(LoggingConfig); StringBuilder.Append(fmt::format("cache write log: {}, cache access log: {}", LoggingConfig.EnableWriteLog ? "true" : "false", LoggingConfig.EnableAccessLog ? "true" : "false")); } } if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) { spdlog::level::level_enum NewLevel = spdlog::level::from_str(Param); spdlog::string_view_t LogLevel = spdlog::level::to_string_view(NewLevel); if (LogLevel != Param) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, fmt::format("Invalid log level '{}'", Param)); } spdlog::set_level(NewLevel); if (StringBuilder.Size() > 0) { StringBuilder.Append(", "); } StringBuilder.Append("loglevel: "); StringBuilder.Append(Param); } return Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, StringBuilder.ToView()); }, HttpVerb::kPost); m_Router.RegisterRoute( "flush", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); if (m_CidStore) { m_CidStore->Flush(); } if (m_CacheStore) { m_CacheStore->Flush(); } if (m_ProjectStore) { m_ProjectStore->Flush(); } HttpReq.WriteResponse(HttpResponseCode::OK); }, HttpVerb::kPost); } HttpAdminService::~HttpAdminService() { } const char* HttpAdminService::BaseUri() const { return "/admin/"; } void HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) { m_Router.HandleRequest(Request); } } // namespace zen