diff options
Diffstat (limited to 'src/zenserver')
| -rw-r--r-- | src/zenserver/config.cpp | 10 | ||||
| -rw-r--r-- | src/zenserver/config.h | 6 | ||||
| -rw-r--r-- | src/zenserver/workspaces/httpworkspaces.cpp | 802 | ||||
| -rw-r--r-- | src/zenserver/workspaces/httpworkspaces.h | 72 | ||||
| -rw-r--r-- | src/zenserver/zenserver.cpp | 15 | ||||
| -rw-r--r-- | src/zenserver/zenserver.h | 4 |
6 files changed, 908 insertions, 1 deletions
diff --git a/src/zenserver/config.cpp b/src/zenserver/config.cpp index ce1b21926..56d14b4a9 100644 --- a/src/zenserver/config.cpp +++ b/src/zenserver/config.cpp @@ -528,6 +528,9 @@ ParseConfigFile(const std::filesystem::path& Path, LuaOptions.Parse(Path, CmdLineResult); + ////// workspaces + LuaOptions.AddOption("workspaces.enabled"sv, ServerOptions.WorksSpacesConfig.Enabled, "workspaces-enabled"sv); + // These have special command line processing so we make sure we export them if they were configured on command line if (!ServerOptions.AuthConfig.OpenIdProviders.empty()) { @@ -1001,6 +1004,13 @@ ParseCliOptions(int argc, char* argv[], ZenServerOptions& ServerOptions) cxxopts::value<bool>(ServerOptions.StatsConfig.Enabled)->default_value("false"), "Enable statsd reporter (localhost:8125)"); + options.add_option("stats", + "", + "workspaces-enabled", + "", + cxxopts::value<bool>(ServerOptions.WorksSpacesConfig.Enabled)->default_value("false"), + "Enable workspaces support with folder sharing"); + try { cxxopts::ParseResult Result; diff --git a/src/zenserver/config.h b/src/zenserver/config.h index 1e44d54c0..fec871a0e 100644 --- a/src/zenserver/config.h +++ b/src/zenserver/config.h @@ -118,6 +118,11 @@ struct ZenStructuredCacheConfig uint64_t MemMaxAgeSeconds = gsl::narrow<uint64_t>(std::chrono::seconds(std::chrono::days(1)).count()); }; +struct ZenWorkspacesConfig +{ + bool Enabled = false; +}; + struct ZenServerOptions { ZenUpstreamCacheConfig UpstreamCacheConfig; @@ -127,6 +132,7 @@ struct ZenServerOptions zen::HttpServerConfig HttpServerConfig; ZenStructuredCacheConfig StructuredCacheConfig; ZenStatsConfig StatsConfig; + ZenWorkspacesConfig WorksSpacesConfig; std::filesystem::path SystemRootDir; // System root directory (used for machine level config) std::filesystem::path DataDir; // Root directory for state (used for testing) std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) diff --git a/src/zenserver/workspaces/httpworkspaces.cpp b/src/zenserver/workspaces/httpworkspaces.cpp new file mode 100644 index 000000000..85403aa78 --- /dev/null +++ b/src/zenserver/workspaces/httpworkspaces.cpp @@ -0,0 +1,802 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include <workspaces/httpworkspaces.h> + +#include <zencore/compactbinarybuilder.h> +#include <zencore/fmtutils.h> +#include <zencore/logging.h> +#include <zencore/trace.h> +#include <zenstore/workspaces.h> +#include <zenutil/basicfile.h> +#include <zenutil/chunkrequests.h> +#include <zenutil/workerpools.h> + +#include <unordered_set> + +namespace zen { +using namespace std::literals; + +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogObj, "fs"sv); + +namespace { + + std::filesystem::path GetPathParameter(HttpServerRequest& ServerRequest, std::string_view Name) + { + if (std::string_view Value = ServerRequest.GetQueryParams().GetValue(Name); !Value.empty()) + { + return std::filesystem::path(HttpServerRequest::Decode(Value)); + } + return {}; + } + + Oid PathToChunkId(const std::filesystem::path& Path) + { + const std::string PathBuffer = reinterpret_cast<const char*>(Path.generic_u8string().c_str()); + BLAKE3 Hash = BLAKE3::HashMemory(PathBuffer.data(), PathBuffer.size()); + Hash.Hash[11] = 7; // FIoChunkType::ExternalFile + return Oid::FromMemory(Hash.Hash); + } + +} // namespace + +HttpWorkspacesService::HttpWorkspacesService(HttpStatsService& StatsService, const FileServeConfig& Cfg, Workspaces& Workspaces) +: m_Log(logging::Get("workspaces")) +, m_StatsService(StatsService) +, m_Config(Cfg) +, m_Workspaces(Workspaces) +{ + Initialize(); +} + +HttpWorkspacesService::~HttpWorkspacesService() +{ + m_StatsService.UnregisterHandler("prj", *this); +} + +const char* +HttpWorkspacesService::BaseUri() const +{ + return "/ws/"; +} + +void +HttpWorkspacesService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + ZEN_LOG_WARN(LogObj, "No route found for {0}", Request.RelativeUri()); + return Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } +} + +void +HttpWorkspacesService::HandleStatsRequest(HttpServerRequest& HttpReq) +{ + ZEN_TRACE_CPU("WorkspacesService::Stats"); + CbObjectWriter Cbo; + + EmitSnapshot("requests", m_HttpRequests, Cbo); + + Cbo.BeginObject("workspaces"); + { + Cbo.BeginObject("workspace"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceReadCount << "writecount" << m_WorkspacesStats.WorkspaceWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("workspaceshare"); + { + Cbo << "readcount" << m_WorkspacesStats.WorkspaceShareReadCount << "writecount" << m_WorkspacesStats.WorkspaceShareWriteCount + << "deletecount" << m_WorkspacesStats.WorkspaceShareDeleteCount; + } + Cbo.EndObject(); + + Cbo.BeginObject("chunk"); + { + Cbo << "hitcount" << m_WorkspacesStats.WorkspaceShareChunkHitCount << "misscount" + << m_WorkspacesStats.WorkspaceShareChunkMissCount; + } + Cbo.EndObject(); + + Cbo << "filescount" << m_WorkspacesStats.WorkspaceShareFilesReadCount; + Cbo << "entriescount" << m_WorkspacesStats.WorkspaceShareEntriesReadCount; + Cbo << "batchcount" << m_WorkspacesStats.WorkspaceShareBatchReadCount; + + Cbo << "requestcount" << m_WorkspacesStats.RequestCount; + Cbo << "badrequestcount" << m_WorkspacesStats.BadRequestCount; + } + Cbo.EndObject(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpWorkspacesService::Initialize() +{ + using namespace std::literals; + + ZEN_LOG_INFO(LogObj, "Initialzing Workspaces Service"); + + m_StatsService.RegisterHandler("ws", *this); + + m_Router.AddPattern("workspace", "([[:xdigit:]]{24})"); + m_Router.AddPattern("share_id", "([[:xdigit:]]{24})"); + m_Router.AddPattern("chunk", "([[:xdigit:]]{24})"); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/files", + [this](HttpRouterRequest& Req) { FilesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}/info", + [this](HttpRouterRequest& Req) { ChunkInfoRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/batch", + [this](HttpRouterRequest& Req) { BatchRequest(Req); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/entries", + [this](HttpRouterRequest& Req) { EntriesRequest(Req); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}/{chunk}", + [this](HttpRouterRequest& Req) { ChunkRequest(Req); }, + HttpVerb::kGet | HttpVerb::kHead); + + m_Router.RegisterRoute( + "{workspace_id}/{share_id}", + [this](HttpRouterRequest& Req) { ShareRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "{workspace_id}", + [this](HttpRouterRequest& Req) { WorkspaceRequest(Req); }, + HttpVerb::kPut | HttpVerb::kGet | HttpVerb::kDelete); + + ReadState(); +} + +std::filesystem::path +HttpWorkspacesService::GetStatePath() const +{ + return m_Config.SystemRootDir / "workspaces"; +} + +void +HttpWorkspacesService::ReadState() +{ + if (!m_Config.SystemRootDir.empty()) + { + m_Workspaces.ReadState(GetStatePath(), [](const std::filesystem::path& Path) { return PathToChunkId(Path); }); + } +} + +void +HttpWorkspacesService::WriteState() +{ + if (!m_Config.SystemRootDir.empty()) + { + m_Workspaces.WriteState(GetStatePath()); + } +} + +void +HttpWorkspacesService::FilesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + + m_WorkspacesStats.WorkspaceShareFilesReadCount++; + + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldnames")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + else + { + const bool FilterClient = ServerRequest.GetQueryParams().GetValue("filter"sv) == "client"sv; + WantedFieldNames.insert("id"); + WantedFieldNames.insert("clientpath"); + if (!FilterClient) + { + WantedFieldNames.insert("serverpath"); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + const bool WantsRawSizeField = WantsAllFields || WantedFieldNames.contains("rawsize"); + const bool WantsSizeField = WantsAllFields || WantedFieldNames.contains("size"); + + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool()); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Response; + Response.BeginArray("files"sv); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + if (WantsSizeField) + { + Response << "size"sv << Entry.Size; + } + if (WantsRawSizeField) + { + Response << "rawsize"sv << Entry.Size; + } + Response.EndObject(); + } + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkInfoRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + Workspaces::ShareFile File = m_Workspaces.GetWorkspaceShareChunkInfo(WorkspaceId, ShareId, ChunkId, GetSmallWorkerPool()); + if (File.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "size"sv << File.Size; + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::BatchRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + IoBuffer Payload = ServerRequest.ReadPayload(); + std::optional<std::vector<RequestChunkEntry>> ChunkRequests = ParseChunkBatchRequest(Payload); + if (!ChunkRequests.has_value()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "batch payload malformed"); + } + m_WorkspacesStats.WorkspaceShareBatchReadCount++; + std::vector<Workspaces::ChunkRequest> Requests; + Requests.reserve(ChunkRequests.value().size()); + std::transform(ChunkRequests.value().begin(), + ChunkRequests.value().end(), + std::back_inserter(Requests), + [](const RequestChunkEntry& Entry) { + return Workspaces::ChunkRequest{.ChunkId = Entry.ChunkId, .Offset = Entry.Offset, .Size = Entry.RequestBytes}; + }); + std::vector<IoBuffer> Chunks = m_Workspaces.GetWorkspaceShareChunks(WorkspaceId, ShareId, Requests, GetSmallWorkerPool()); + if (Chunks.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + for (const IoBuffer& Buffer : Chunks) + { + if (Buffer) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + } + else + { + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + } + } + std::vector<IoBuffer> Response = BuildChunkBatchResponse(ChunkRequests.value(), Chunks); + if (!Response.empty()) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Response); + } + return ServerRequest.WriteResponse(HttpResponseCode::InternalServerError, + HttpContentType::kText, + fmt::format("failed formatting response for batch of {} chunks", Chunks.size())); +} + +void +HttpWorkspacesService::EntriesRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + std::string_view OpKey = ServerRequest.GetQueryParams().GetValue("opkey"sv); + if (!OpKey.empty() && OpKey != "file_manifest") + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + std::unordered_set<std::string> WantedFieldNames; + if (auto FieldFilter = HttpServerRequest::Decode(ServerRequest.GetQueryParams().GetValue("fieldfilter")); !FieldFilter.empty()) + { + if (FieldFilter != "*") // Get all - empty FieldFilter equal getting all fields + { + ForEachStrTok(FieldFilter, ',', [&](std::string_view FieldName) { + WantedFieldNames.insert(std::string(FieldName)); + return true; + }); + } + } + + bool Refresh = false; + if (auto RefreshStr = ServerRequest.GetQueryParams().GetValue("refresh"); !RefreshStr.empty()) + { + Refresh = StrCaseCompare(std::string(RefreshStr).c_str(), "true") == 0; + } + + m_WorkspacesStats.WorkspaceShareEntriesReadCount++; + std::optional<std::vector<Workspaces::ShareFile>> Files = + m_Workspaces.GetWorkspaceShareFiles(WorkspaceId, ShareId, Refresh, GetSmallWorkerPool()); + if (!Files.has_value()) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + const bool WantsAllFields = WantedFieldNames.empty(); + + const bool WantsIdField = WantsAllFields || WantedFieldNames.contains("id"); + const bool WantsClientPathField = WantsAllFields || WantedFieldNames.contains("clientpath"); + const bool WantsServerPathField = WantsAllFields || WantedFieldNames.contains("serverpath"); + + CbObjectWriter Response; + + if (OpKey.empty()) + { + Response.BeginArray("entries"sv); + Response.BeginObject(); + } + else + { + Response.BeginObject("entry"sv); + } + { + // Synthesize a fake op + Response << "key" + << "file_manifest"; + + Response.BeginArray("files"); + { + for (const Workspaces::ShareFile& Entry : Files.value()) + { + Response.BeginObject(); + { + if (WantsIdField) + { + Response << "id"sv << Entry.Id; + } + if (WantsServerPathField) + { + Response << "serverpath"sv << Entry.RelativePath; + } + if (WantsClientPathField) + { + Response << "clientpath"sv << Entry.RelativePath; + } + } + Response.EndObject(); + } + } + Response.EndArray(); + } + + if (OpKey.empty()) + { + Response.EndObject(); + Response.EndArray(); + } + else + { + Response.EndObject(); + } + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); +} + +void +HttpWorkspacesService::ChunkRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + const Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + const Oid ChunkId = Oid::TryFromHexString(Req.GetCapture(3)); + if (ChunkId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid chunk id '{}'", Req.GetCapture(3))); + } + + uint64_t Offset = 0; + uint64_t Size = ~(0ull); + if (auto OffsetParm = ServerRequest.GetQueryParams().GetValue("offset"); OffsetParm.empty() == false) + { + if (auto OffsetVal = ParseInt<uint64_t>(OffsetParm)) + { + Offset = OffsetVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid offset parameter '{}'", OffsetParm)); + } + } + + if (auto SizeParm = ServerRequest.GetQueryParams().GetValue("size"); SizeParm.empty() == false) + { + if (auto SizeVal = ParseInt<uint64_t>(SizeParm)) + { + Size = SizeVal.value(); + } + else + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid size parameter '{}'", SizeParm)); + } + } + + std::vector<IoBuffer> Response = m_Workspaces.GetWorkspaceShareChunks( + WorkspaceId, + ShareId, + std::vector<Workspaces::ChunkRequest>{Workspaces::ChunkRequest{.ChunkId = ChunkId, .Offset = Offset, .Size = Size}}, + GetSmallWorkerPool()); + if (!Response.empty() && Response[0]) + { + m_WorkspacesStats.WorkspaceShareChunkHitCount++; + if (Response[0].GetSize() == 0) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response[0].GetContentType(), Response); + } + m_WorkspacesStats.WorkspaceShareChunkMissCount++; + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); +} + +void +HttpWorkspacesService::ShareRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + const Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + Oid ShareId = Oid::TryFromHexString(Req.GetCapture(2)); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path SharePath = GetPathParameter(ServerRequest, "share_path"sv); + if (SharePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'share_path' parameter"); + } + if (Req.GetCapture(2) == Oid::Zero.ToString()) + { + // Synthesize Id + ShareId = PathToChunkId(SharePath); + ZEN_INFO("Generated workspace id from path '{}': {}", SharePath, ShareId); + } + else if (ShareId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid share id '{}'", Req.GetCapture(2))); + } + m_WorkspacesStats.WorkspaceShareWriteCount++; + if (m_Workspaces.GetWorkspaceInfo(WorkspaceId).Config.Id != WorkspaceId) + { + return ServerRequest.WriteResponse(HttpResponseCode::NotFound, + HttpContentType::kText, + fmt::format("Workspace '{}' does not exist", WorkspaceId)); + } + bool OK = m_Workspaces.AddWorkspaceShare(WorkspaceId, {ShareId, SharePath}, [](const std::filesystem::path& Path) { + return PathToChunkId(Path); + }); + if (OK) + { + WriteState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", ShareId)); + } + else + { + Workspaces::WorkspaceShareConfiguration Config = m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); + if (Config.Id == ShareId) + { + if (Config.SharePath == SharePath) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, fmt::format("{}", ShareId)); + } + } + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace share '{}' already exist in workspace '{}'", ShareId, WorkspaceId)); + } + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceShareReadCount++; + Workspaces::WorkspaceShareConfiguration Config = m_Workspaces.GetWorkspaceShareConfiguration(WorkspaceId, ShareId); + if (Config.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "id" << Config.Id; + Response << "share_path" << Config.SharePath.string(); // utf8? + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceShareDeleteCount++; + bool Deleted = m_Workspaces.RemoveWorkspaceShare(WorkspaceId, ShareId); + if (Deleted) + { + WriteState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +void +HttpWorkspacesService::WorkspaceRequest(HttpRouterRequest& Req) +{ + HttpServerRequest& ServerRequest = Req.ServerRequest(); + Oid WorkspaceId = Oid::TryFromHexString(Req.GetCapture(1)); + switch (ServerRequest.RequestVerb()) + { + case HttpVerb::kPut: + { + std::filesystem::path WorkspacePath = GetPathParameter(ServerRequest, "root_path"sv); + if (WorkspacePath.empty()) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid 'root_path' parameter"); + } + if (Req.GetCapture(1) == Oid::Zero.ToString()) + { + // Synthesize Id + WorkspaceId = PathToChunkId(WorkspacePath); + ZEN_INFO("Generated workspace id from path '{}': {}", WorkspacePath, WorkspaceId); + } + else if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceWriteCount++; + bool OK = m_Workspaces.AddWorkspace({WorkspaceId, WorkspacePath}); + if (OK) + { + WriteState(); + return ServerRequest.WriteResponse(HttpResponseCode::Created, HttpContentType::kText, fmt::format("{}", WorkspaceId)); + } + else + { + Workspaces::WorkspaceInfo Info = m_Workspaces.GetWorkspaceInfo(WorkspaceId); + if (Info.Config.Id == WorkspaceId) + { + if (Info.Config.RootPath == WorkspacePath) + { + return ServerRequest.WriteResponse(HttpResponseCode::OK, + HttpContentType::kText, + fmt::format("{}", WorkspaceId)); + } + } + return ServerRequest.WriteResponse( + HttpResponseCode::Conflict, + HttpContentType::kText, + fmt::format("Workspace {} already exists with root path '{}'", WorkspaceId, Info.Config.RootPath)); + } + } + case HttpVerb::kGet: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceReadCount++; + Workspaces::WorkspaceInfo Info = m_Workspaces.GetWorkspaceInfo(WorkspaceId); + if (Info.Config.Id != Oid::Zero) + { + CbObjectWriter Response; + Response << "id" << Info.Config.Id; + Response << "root_path" << Info.Config.RootPath.string(); // utf8? + Response.BeginArray("shares"); + for (const Workspaces::WorkspaceShareConfiguration& ShareConfig : Info.Shares) + { + Response.BeginObject(); + { + Response << "id" << ShareConfig.Id; + Response << "share_path" << ShareConfig.SharePath.string(); // utf8? + } + Response.EndObject(); + } + Response.EndArray(); + + return ServerRequest.WriteResponse(HttpResponseCode::OK, Response.Save()); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + case HttpVerb::kDelete: + { + if (WorkspaceId == Oid::Zero) + { + m_WorkspacesStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Invalid workspace id '{}'", Req.GetCapture(1))); + } + m_WorkspacesStats.WorkspaceDeleteCount++; + bool Deleted = m_Workspaces.RemoveWorkspace(WorkspaceId); + if (Deleted) + { + WriteState(); + return ServerRequest.WriteResponse(HttpResponseCode::OK); + } + return ServerRequest.WriteResponse(HttpResponseCode::NotFound); + } + } +} + +} // namespace zen diff --git a/src/zenserver/workspaces/httpworkspaces.h b/src/zenserver/workspaces/httpworkspaces.h new file mode 100644 index 000000000..cfd23e7ba --- /dev/null +++ b/src/zenserver/workspaces/httpworkspaces.h @@ -0,0 +1,72 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include <zencore/stats.h> +#include <zenhttp/httpserver.h> +#include <zenhttp/httpstats.h> + +namespace zen { + +class Workspaces; + +struct FileServeConfig +{ + std::filesystem::path SystemRootDir; +}; + +class HttpWorkspacesService final : public HttpService, public IHttpStatsProvider +{ +public: + HttpWorkspacesService(HttpStatsService& StatsService, const FileServeConfig& Cfg, Workspaces& Workspaces); + virtual ~HttpWorkspacesService(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + +private: + struct WorkspacesStats + { + std::atomic_uint64_t WorkspaceReadCount{}; + std::atomic_uint64_t WorkspaceWriteCount{}; + std::atomic_uint64_t WorkspaceDeleteCount{}; + std::atomic_uint64_t WorkspaceShareReadCount{}; + std::atomic_uint64_t WorkspaceShareWriteCount{}; + std::atomic_uint64_t WorkspaceShareDeleteCount{}; + std::atomic_uint64_t WorkspaceShareFilesReadCount{}; + std::atomic_uint64_t WorkspaceShareEntriesReadCount{}; + std::atomic_uint64_t WorkspaceShareBatchReadCount{}; + std::atomic_uint64_t WorkspaceShareChunkHitCount{}; + std::atomic_uint64_t WorkspaceShareChunkMissCount{}; + std::atomic_uint64_t RequestCount{}; + std::atomic_uint64_t BadRequestCount{}; + }; + + inline LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + void Initialize(); + std::filesystem::path GetStatePath() const; + void ReadState(); + void WriteState(); + + void FilesRequest(HttpRouterRequest& Req); + void ChunkInfoRequest(HttpRouterRequest& Req); + void BatchRequest(HttpRouterRequest& Req); + void EntriesRequest(HttpRouterRequest& Req); + void ChunkRequest(HttpRouterRequest& Req); + void ShareRequest(HttpRouterRequest& Req); + void WorkspaceRequest(HttpRouterRequest& Req); + + HttpStatsService& m_StatsService; + const FileServeConfig m_Config; + HttpRequestRouter m_Router; + Workspaces& m_Workspaces; + WorkspacesStats m_WorkspacesStats; + metrics::OperationTiming m_HttpRequests; +}; + +} // namespace zen diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 0909c26e9..9f24960bd 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include <zenhttp/httpserver.h> #include <zenstore/cidstore.h> #include <zenstore/scrubcontext.h> +#include <zenstore/workspaces.h> #include <zenutil/basicfile.h> #include <zenutil/workerpools.h> #include <zenutil/zenserverprocess.h> @@ -226,6 +227,13 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_ProjectStore = new ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager, *m_JobQueue); m_HttpProjectService.reset(new HttpProjectService{*m_CidStore, m_ProjectStore, m_StatsService, *m_AuthMgr}); + if (ServerOptions.WorksSpacesConfig.Enabled) + { + m_Workspaces.reset(new Workspaces()); + m_HttpWorkspacesService.reset( + new HttpWorkspacesService(m_StatsService, {.SystemRootDir = ServerOptions.SystemRootDir}, *m_Workspaces)); + } + if (ServerOptions.StructuredCacheConfig.Enabled) { InitializeStructuredCache(ServerOptions); @@ -246,6 +254,11 @@ ZenServer::Initialize(const ZenServerOptions& ServerOptions, ZenServerState::Zen m_Http->RegisterService(*m_HttpProjectService); } + if (m_HttpWorkspacesService) + { + m_Http->RegisterService(*m_HttpWorkspacesService); + } + m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); if (m_FrontendService) @@ -761,6 +774,8 @@ ZenServer::Cleanup() m_UpstreamCache.reset(); m_CacheStore = {}; + m_HttpWorkspacesService.reset(); + m_Workspaces.reset(); m_HttpProjectService.reset(); m_ProjectStore = {}; m_CidStore.reset(); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index 0bab4e0a7..b9d12689d 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -34,6 +34,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "stats/statsreporter.h" #include "upstream/upstream.h" #include "vfs/vfsservice.h" +#include "workspaces/httpworkspaces.h" #ifndef ZEN_APP_NAME # define ZEN_APP_NAME "Unreal Zen Storage Server" @@ -55,7 +56,6 @@ public: int Initialize(const ZenServerOptions& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); void InitializeState(const ZenServerOptions& ServerOptions); void InitializeStructuredCache(const ZenServerOptions& ServerOptions); - void InitializeCompute(const ZenServerOptions& ServerOptions); void Run(); void RequestExit(int ExitCode); @@ -131,6 +131,8 @@ private: #endif RefPtr<ProjectStore> m_ProjectStore; std::unique_ptr<HttpProjectService> m_HttpProjectService; + std::unique_ptr<Workspaces> m_Workspaces; + std::unique_ptr<HttpWorkspacesService> m_HttpWorkspacesService; std::unique_ptr<UpstreamCache> m_UpstreamCache; std::unique_ptr<HttpUpstreamService> m_UpstreamService; std::unique_ptr<HttpStructuredCacheService> m_StructuredCacheService; |