diff options
| author | Stefan Boberg <[email protected]> | 2023-06-16 17:02:23 +0200 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-06-16 17:02:23 +0200 |
| commit | 86418ed57c677ffc8ce314acfcf9b9f1eab3586b (patch) | |
| tree | d197ea40f2e8255338c806a27e82ae289a33bf68 /src | |
| parent | added ZenServerInstance::SpawnServerAndWait (#334) (diff) | |
| download | zen-86418ed57c677ffc8ce314acfcf9b9f1eab3586b.tar.xz zen-86418ed57c677ffc8ce314acfcf9b9f1eab3586b.zip | |
file share support (#328)
this change adds a serve command to the zen CLI. This can be used to establish links to a set of files which may be served to clients via the project store interface:
```cmd> zen serve Lyra/WindowsClient d:\temp_share\StagedBuilds\WindowsClient```
with the appropriate changes in UE you may then start an instance of the runtime and have it load all files via the remote file connection:
```
Lyra\Binaries\LyraClient.exe ../../../Lyra/Lyra.uproject -pak -basedir=D:\temp_share\StagedBuilds\WindowsClient/Lyra/Binaries/Win64 -Mount=Lyra/WindowsClient
```
Diffstat (limited to 'src')
| -rw-r--r-- | src/zen/cmds/serve.cpp | 241 | ||||
| -rw-r--r-- | src/zen/cmds/serve.h | 28 | ||||
| -rw-r--r-- | src/zen/zen.cpp | 40 | ||||
| -rw-r--r-- | src/zen/zen.h | 1 | ||||
| -rw-r--r-- | src/zenserver/projectstore/httpprojectstore.cpp | 39 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.cpp | 33 | ||||
| -rw-r--r-- | src/zenserver/projectstore/projectstore.h | 13 |
7 files changed, 353 insertions, 42 deletions
diff --git a/src/zen/cmds/serve.cpp b/src/zen/cmds/serve.cpp new file mode 100644 index 000000000..3a50adb69 --- /dev/null +++ b/src/zen/cmds/serve.cpp @@ -0,0 +1,241 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "serve.h" + +#include <zencore/blake3.h> +#include <zencore/compactbinarybuilder.h> +#include <zencore/compactbinarypackage.h> +#include <zencore/filesystem.h> +#include <zencore/fmtutils.h> +#include <zenhttp/httpclient.h> +#include <zenutil/zenserverprocess.h> + +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> // TEMPORARY HACK +#endif + +namespace zen { + +using namespace std::literals; + +ServeCommand::ServeCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), "<hosturl>"); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>"); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>"); + m_Options.add_option("", "", "path", "Root path to directory", cxxopts::value(m_RootPath), "<rootpath>"); + + m_Options.parse_positional({"project", "path"}); + m_Options.positional_help("[<projectid> <rootpath>]"); +} + +ServeCommand::~ServeCommand() +{ +} + +int +ServeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + if (m_ProjectName.empty()) + { + throw zen::OptionParseException("command requires a project"); + } + + if (m_OplogName.empty()) + { + if (auto pos = m_ProjectName.find_first_of('/'); pos != std::string::npos) + { + m_OplogName = m_ProjectName.substr(pos + 1); + m_ProjectName = m_ProjectName.substr(0, pos); + } + else + { + throw zen::OptionParseException("command requires an oplog"); + } + } + + if (m_RootPath.empty()) + { + throw zen::OptionParseException("command requires a root path"); + } + + if (!std::filesystem::exists(m_RootPath) || !std::filesystem::is_directory(m_RootPath)) + { + throw zen::OptionParseException(fmt::format("path must exist and must be a directory: '{}'", m_RootPath)); + } + + uint16_t ServerPort = 0; + m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); + + ZenServerEnvironment ServerEnvironment; + std::optional<ZenServerInstance> ServerInstance; + + if (m_HostName.empty()) + { + // Spawn a server + + try + { + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + + ServerEnvironment.Initialize(ExePath.parent_path()); + ServerInstance.emplace(ServerEnvironment); + ServerInstance->SetOwnerPid(zen::GetCurrentProcessId()); + ServerInstance->SpawnServerAndWait(ServerPort); + } + catch (std::exception& Ex) + { + ZEN_CONSOLE("failed to spawn server on port {}: '{}'", ServerPort, Ex.what()); + + throw zen::OptionParseException("unable to resolve server specification (even after spawning server)"); + } + } + else + { + std::filesystem::path ExePath = zen::GetRunningExecutablePath(); + + ServerEnvironment.Initialize(ExePath.parent_path()); + ServerInstance.emplace(ServerEnvironment); + ServerInstance->DisableShutdownOnDestroy(); + ServerInstance->AttachToRunningServer(); + } + + if (ServerInstance) + { + ZEN_CONSOLE("base uri: {}", ServerInstance->GetBaseUri()); + } + + // Generate manifest for tree + + FileSystemTraversal Traversal; + + struct FsVisitor : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + { + std::filesystem::path ServerPath = std::filesystem::relative(Parent / File, RootPath); + std::string ServerPathString = reinterpret_cast<const char*>(ServerPath.generic_u8string().c_str()); + + if (ServerPathString.starts_with("./")) + { + ServerPathString = ServerPathString.substr(2); + } + + Files.emplace_back(FileEntry{ServerPathString, ServerPathString, FileSize}); + } + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } + + struct FileEntry + { + std::string FilePath; + std::string ClientFilePath; + uint64_t FileSize; + }; + + std::filesystem::path RootPath; + std::vector<FileEntry> Files; + }; + + FsVisitor Visitor; + Visitor.RootPath = m_RootPath; + Traversal.TraverseFileSystem(m_RootPath, Visitor); + + CbObjectWriter Cbo; + + Cbo << "key" + << "file_manifest"; + + Cbo.BeginArray("files"); + + for (const FsVisitor::FileEntry& Entry : Visitor.Files) + { + ZEN_CONSOLE("file: {}", Entry.FilePath); + + Cbo.BeginObject(); + + BLAKE3 Hash = BLAKE3::HashMemory(Entry.ClientFilePath.data(), Entry.ClientFilePath.size()); + Hash.Hash[11] = 7; // FIoChunkType::ExternalFile + Oid FileChunkId = Oid::FromMemory(Hash.Hash); + + Cbo << "id"sv << FileChunkId; + Cbo << "serverpath"sv << Entry.FilePath; + Cbo << "clientpath"sv << Entry.ClientFilePath; + + Cbo.EndObject(); + } + + Cbo.EndArray(); + + CbObject Manifest = Cbo.Save(); + + // Persist manifest + + const std::string ProjectUri = fmt::format("/prj/{}", m_ProjectName); + const std::string ProjectOplogUri = fmt::format("/prj/{}/oplog/{}", m_ProjectName, m_OplogName); + + HttpClient Client(m_HostName); + + // Ensure project exists + + if (HttpClient::Response ProjectResponse = Client.Get(ProjectUri); !ProjectResponse) + { + // Create project + + CbObjectWriter Project; + + Project << "root" << m_RootPath; + + if (auto NewProjectResponse = Client.Post(ProjectUri, Project.Save()); !NewProjectResponse) + { + // TODO: include details + throw std::runtime_error("failed to create project"); + } + } + + // Ensure oplog exists + + if (HttpClient::Response OplogResponse = Client.Get(ProjectOplogUri); !OplogResponse) + { + // Create oplog + + CbObjectWriter Oplog; + + if (auto NewOplogResponse = Client.Post(ProjectOplogUri, Oplog.Save()); !NewOplogResponse) + { + // TODO: include details + throw std::runtime_error("failed to create oplog"); + } + } + + // Append manifest + + const std::string Uri = fmt::format("/prj/{}/oplog/{}/new", m_ProjectName, m_OplogName); + + HttpClient::Response HttpResponse = Client.Post(Uri, Manifest); + + if (!HttpResponse) + { + ZEN_CONSOLE("error: failed to append manifest!"); + + return 1; + } + + ZEN_CONSOLE("ok serving files now"); + +#if ZEN_PLATFORM_WINDOWS + _getch(); // TEMPORARY HACK +#endif + + return 0; +} + +} // namespace zen diff --git a/src/zen/cmds/serve.h b/src/zen/cmds/serve.h new file mode 100644 index 000000000..007038d84 --- /dev/null +++ b/src/zen/cmds/serve.h @@ -0,0 +1,28 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +namespace zen { + +/** File serving + */ +class ServeCommand : public ZenCmdBase +{ +public: + ServeCommand(); + ~ServeCommand(); + + virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{"serve", "Serve files from a tree"}; + std::string m_HostName; + std::string m_ProjectName; + std::string m_OplogName; + std::string m_RootPath; +}; + +} // namespace zen diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 464b2817c..a2ab31254 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -14,6 +14,7 @@ #include "cmds/projectstore.h" #include "cmds/rpcreplay.h" #include "cmds/scrub.h" +#include "cmds/serve.h" #include "cmds/status.h" #include "cmds/top.h" #include "cmds/up.h" @@ -207,6 +208,7 @@ main(int argc, char** argv) RpcStartRecordingCommand RpcStartRecordingCmd; RpcStopRecordingCommand RpcStopRecordingCmd; ScrubCommand ScrubCmd; + ServeCommand ServeCmd; SnapshotOplogCommand SnapshotOplogCmd; StatusCommand StatusCmd; TopCommand TopCmd; @@ -238,6 +240,7 @@ main(int argc, char** argv) {"oplog-create", &CreateOplogCmd, "Create a project oplog"}, {"oplog-export", &ExportOplogCmd, "Export project store oplog"}, {"oplog-import", &ImportOplogCmd, "Import project store oplog"}, + {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"}, {"print", &PrintCmd, "Print compact binary object"}, {"printpackage", &PrintPkgCmd, "Print compact binary package"}, {"project-create", &CreateProjectCmd, "Create a project"}, @@ -250,7 +253,7 @@ main(int argc, char** argv) {"rpc-record-start", &RpcStartRecordingCmd, "Replays a previously recorded session of rpc requests"}, {"rpc-record-stop", &RpcStopRecordingCmd, "Starts recording of cache rpc requests on a host"}, {"scrub", &ScrubCmd, "Scrub zen storage (verify data integrity)"}, - {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"}, + {"serve", &ServeCmd, "Serve files from a directory"}, {"status", &StatusCmd, "Show zen status"}, {"top", &TopCmd, "Monitor zen server activity"}, {"up", &UpCmd, "Bring zen server up"}, @@ -437,20 +440,24 @@ main(int argc, char** argv) } std::string -ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) +ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort) { if (InHostSpec.empty()) { + // If no host is specified then look to see if we have an instance + // running on this host and use that as the default to interact with + zen::ZenServerState Servers; if (Servers.InitializeReadOnly()) { - std::string ResolvedSpec = InHostSpec; + std::string ResolvedSpec; Servers.Snapshot([&](const zen::ZenServerState::ZenServerEntry& Entry) { if (ResolvedSpec.empty()) { - ResolvedSpec = fmt::format("http://localhost:{}", Entry.EffectiveListenPort); + ResolvedSpec = fmt::format("http://localhost:{}", Entry.EffectiveListenPort); + OutEffectivePort = Entry.EffectiveListenPort; } }); @@ -458,5 +465,30 @@ ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) } } + // Parse out port from the specification provided, to be consistent with + // the auto-discovery logic above. + + std::string_view PortSpec(InHostSpec); + if (size_t PrefixIndex = PortSpec.find_last_of(":"); PrefixIndex != std::string_view::npos) + { + PortSpec.remove_prefix(PrefixIndex + 1); + + std::optional<uint16_t> EffectivePort = zen::ParseInt<uint16_t>(PortSpec); + + if (EffectivePort) + { + OutEffectivePort = EffectivePort.value(); + } + } + + // note: We should consider adding validation/normalization of the provided spec here + return InHostSpec; } + +std::string +ZenCmdBase::ResolveTargetHostSpec(const std::string& InHostSpec) +{ + uint16_t Dummy = 0; + return ResolveTargetHostSpec(InHostSpec, /* out */ Dummy); +} diff --git a/src/zen/zen.h b/src/zen/zen.h index 84f4e1333..7258c10ce 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -36,4 +36,5 @@ public: static std::string FormatHttpResponse(const cpr::Response& Response); static int MapHttpToCommandReturnCode(const cpr::Response& Response); static std::string ResolveTargetHostSpec(const std::string& InHostSpec); + static std::string ResolveTargetHostSpec(const std::string& InHostSpec, uint16_t& OutEffectivePort); }; diff --git a/src/zenserver/projectstore/httpprojectstore.cpp b/src/zenserver/projectstore/httpprojectstore.cpp index b0efb6bec..185f2783d 100644 --- a/src/zenserver/projectstore/httpprojectstore.cpp +++ b/src/zenserver/projectstore/httpprojectstore.cpp @@ -758,14 +758,23 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, if (!legacy::TryLoadCbPackage(Package, Payload, &UniqueBuffer::Alloc, &Resolver)) { - std::filesystem::path BadPackagePath = - Oplog.TempPath() / "bad_packages"sv / fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); + if (CbObject Core = LoadCompactBinaryObject(Payload)) + { + Package.SetObject(Core); + } + else + { + std::filesystem::path BadPackagePath = Oplog.TempPath() / "bad_packages"sv / + fmt::format("session{}_request{}"sv, HttpReq.SessionId(), HttpReq.RequestId()); - ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath); + ZEN_WARN("Received malformed package! Saving payload to '{}'", BadPackagePath); - WriteFile(BadPackagePath, Payload); + WriteFile(BadPackagePath, Payload); - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, "Invalid package"); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + u8"request body must be a compact binary object or package in legacy format"); + } } if (!IsValid) @@ -1056,20 +1065,20 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects, return HttpReq.WriteResponse(HttpResponseCode::InsufficientStorage); } - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject Params = LoadCompactBinaryObject(Payload); - std::string_view Id = Params["id"sv].AsString(); - std::string_view Root = Params["root"sv].AsString(); - std::string_view EngineRoot = Params["engine"sv].AsString(); - std::string_view ProjectRoot = Params["project"sv].AsString(); - std::string_view ProjectFilePath = Params["projectfile"sv].AsString(); + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject Params = LoadCompactBinaryObject(Payload); + std::string_view Root = Params["root"sv].AsString(); // Workspace root (i.e `D:/UE5/`) + std::string_view EngineRoot = Params["engine"sv].AsString(); // Engine root (i.e `D:/UE5/Engine`) + std::string_view ProjectRoot = + Params["project"sv].AsString(); // Project root directory (i.e `D:/UE5/Samples/Games/Lyra`) + std::string_view ProjectFilePath = + Params["projectfile"sv].AsString(); // Project file path (i.e `D:/UE5/Samples/Games/Lyra/Lyra.uproject`) const std::filesystem::path BasePath = m_ProjectStore->BasePath() / ProjectId; m_ProjectStore->NewProject(BasePath, ProjectId, Root, EngineRoot, ProjectRoot, ProjectFilePath); - ZEN_INFO("established project - {} (id: '{}', roots: '{}', '{}', '{}', '{}'{})", + ZEN_INFO("established project (id: '{}', roots: '{}', '{}', '{}', '{}'{})", ProjectId, - Id, Root, EngineRoot, ProjectRoot, @@ -1496,4 +1505,4 @@ HttpProjectService::HandleStatsRequest(HttpServerRequest& HttpReq) return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } -} // namespace zen
\ No newline at end of file +} // namespace zen diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp index 6ca43af0e..184376c39 100644 --- a/src/zenserver/projectstore/projectstore.cpp +++ b/src/zenserver/projectstore/projectstore.cpp @@ -574,8 +574,7 @@ ProjectStore::Oplog::ReplayLog() { return; } - m_Storage->ReplayLog( - [&](CbObject Op, const OplogEntry& OpEntry) { RegisterOplogEntry(OplogLock, GetMapping(Op), OpEntry, kUpdateReplay); }); + m_Storage->ReplayLog([&](CbObject Op, const OplogEntry& OpEntry) { RegisterOplogEntry(OplogLock, GetMapping(Op), OpEntry); }); } IoBuffer @@ -774,21 +773,20 @@ ProjectStore::Oplog::AddFileMapping(const RwLock::ExclusiveLockScope&, std::string_view ServerPath, std::string_view ClientPath) { + FileMapEntry Entry; + if (Hash != IoHash::Zero) { m_ChunkMap.insert_or_assign(FileId, Hash); } + else + { + Entry.ServerPath = ServerPath; + } - FileMapEntry Entry; - Entry.ServerPath = ServerPath; Entry.ClientPath = ClientPath; m_FileMap[FileId] = std::move(Entry); - - if (Hash != IoHash::Zero) - { - m_ChunkMap.insert_or_assign(FileId, Hash); - } } void @@ -858,8 +856,7 @@ ProjectStore::Oplog::GetMapping(CbObject Core) Oid Id = FileObj["id"sv].AsObjectId(); IoHash Hash = FileObj["data"sv].AsBinaryAttachment(); - Result.Files.emplace_back( - OplogEntryMapping::FileMapping{OplogEntryMapping::Mapping{Id, Hash}, std::string(ServerPath), std::string(ClientPath)}); + Result.Files.emplace_back(OplogEntryMapping::FileMapping{Id, Hash, std::string(ServerPath), std::string(ClientPath)}); ZEN_DEBUG("file {} -> {}, ServerPath: {}, ClientPath: {}", Id, Hash, ServerPath, ClientPath); } @@ -881,13 +878,10 @@ ProjectStore::Oplog::GetMapping(CbObject Core) uint32_t ProjectStore::Oplog::RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock, const OplogEntryMapping& OpMapping, - const OplogEntry& OpEntry, - UpdateType TypeOfUpdate) + const OplogEntry& OpEntry) { ZEN_TRACE_CPU("ProjectStore::Oplog::RegisterOplogEntry"); - ZEN_UNUSED(TypeOfUpdate); - // For now we're assuming the update is all in-memory so we can hold an exclusive lock without causing // too many problems. Longer term we'll probably want to ensure we can do concurrent updates however @@ -986,7 +980,7 @@ ProjectStore::Oplog::AppendNewOplogEntry(CbObject Core) const OplogEntry OpEntry = m_Storage->AppendOp(Buffer, OpCoreHash, KeyHash); RwLock::ExclusiveLockScope OplogLock(m_OplogLock); - const uint32_t EntryId = RegisterOplogEntry(OplogLock, Mapping, OpEntry, kUpdateNewEntry); + const uint32_t EntryId = RegisterOplogEntry(OplogLock, Mapping, OpEntry); return EntryId; } @@ -1904,7 +1898,7 @@ ProjectStore::GetProjectFiles(const std::string_view ProjectId, const std::strin Response.BeginObject(); Response << "id"sv << Id; Response << "clientpath"sv << ClientPath; - if (!FilterClient) + if (!FilterClient && !ServerPath.empty()) { Response << "serverpath"sv << ServerPath; } @@ -2519,6 +2513,11 @@ ProjectStore::Rpc(HttpServerRequest& HttpReq, return true; } + else if (Field.GetName() == "serverpath"sv) + { + // omit this field as it's not relevant if there is a hash + return true; + } return false; }); diff --git a/src/zenserver/projectstore/projectstore.h b/src/zenserver/projectstore/projectstore.h index 8b5369e98..c9816ec55 100644 --- a/src/zenserver/projectstore/projectstore.h +++ b/src/zenserver/projectstore/projectstore.h @@ -166,11 +166,15 @@ public: Oid Id; IoHash Hash; }; - struct FileMapping : public Mapping + + struct FileMapping { - std::string ServerPath; + Oid Id; + IoHash Hash; // This is either zero or a cid + std::string ServerPath; // If Hash is valid then this should be empty std::string ClientPath; }; + std::vector<Mapping> Chunks; std::vector<Mapping> Meta; std::vector<FileMapping> Files; @@ -184,10 +188,7 @@ public: * * Returns the oplog LSN assigned to the new entry, or kInvalidOp if the entry is rejected */ - uint32_t RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock, - const OplogEntryMapping& OpMapping, - const OplogEntry& OpEntry, - UpdateType TypeOfUpdate); + uint32_t RegisterOplogEntry(RwLock::ExclusiveLockScope& OplogLock, const OplogEntryMapping& OpMapping, const OplogEntry& OpEntry); void AddFileMapping(const RwLock::ExclusiveLockScope& OplogLock, Oid FileId, |