diff options
| author | Stefan Boberg <[email protected]> | 2023-01-24 22:43:18 +0100 |
|---|---|---|
| committer | Stefan Boberg <[email protected]> | 2023-01-24 22:48:03 +0100 |
| commit | 4eed3f1730c6c018d5970edb90921ba3e44f7d87 (patch) | |
| tree | 910e61323e45748a3a3560b6f0619ba097e858db | |
| parent | removed HttpLaunchService and related code (diff) | |
| download | zen-4eed3f1730c6c018d5970edb90921ba3e44f7d87.tar.xz zen-4eed3f1730c6c018d5970edb90921ba3e44f7d87.zip | |
removed HttpLaunchService and related code
this was used for testing but is no longer wanted in this form
| -rw-r--r-- | zen/cmds/run.cpp | 189 | ||||
| -rw-r--r-- | zen/cmds/run.h | 30 | ||||
| -rw-r--r-- | zen/xmake.lua | 4 | ||||
| -rw-r--r-- | zen/zen.cpp | 15 | ||||
| -rw-r--r-- | zenserver-test/zenserver-test.cpp | 175 | ||||
| -rw-r--r-- | zenserver/testing/launch.cpp | 565 | ||||
| -rw-r--r-- | zenserver/testing/launch.h | 48 | ||||
| -rw-r--r-- | zenserver/zenserver.cpp | 30 |
8 files changed, 6 insertions, 1050 deletions
diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp deleted file mode 100644 index 6ba74e2fd..000000000 --- a/zen/cmds/run.cpp +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#define _SILENCE_ALL_CXX17_DEPRECATION_WARNINGS // for <cstdbool> include warning triggered by cpr - -#include "run.h" - -#if ZEN_WITH_EXEC_SERVICES - -# include <zencore/compactbinarybuilder.h> -# include <zencore/except.h> -# include <zencore/filesystem.h> -# include <zencore/fmtutils.h> -# include <zencore/iohash.h> -# include <zencore/logging.h> -# include <zencore/stream.h> -# include <zencore/string.h> -# include <zencore/timer.h> -# include <zenutil/zenserverprocess.h> - -# include <filesystem> - -ZEN_THIRD_PARTY_INCLUDES_START -# include <cpr/cpr.h> -ZEN_THIRD_PARTY_INCLUDES_END - -# if ZEN_PLATFORM_WINDOWS -# pragma comment(lib, "Crypt32.lib") -# pragma comment(lib, "Wldap32.lib") -# pragma comment(lib, "Ws2_32.lib") -# endif - -////////////////////////////////////////////////////////////////////////// - -namespace zen { - -using namespace std::literals; - -RunCommand::RunCommand() -{ - m_Options.add_options()("h,host", "Host to run on", cxxopts::value<std::string>(m_TargetHost))("d,dir", - "Tree to run", - cxxopts::value<std::string>(m_ExeTree)); -} - -RunCommand::~RunCommand() = default; - -void -CreateTreeManifest(std::filesystem::path RootPath) -{ - ZEN_UNUSED(RootPath); -} - -int -RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) -{ - if (GlobalOptions.PassthroughV.empty()) - { - throw cxxopts::OptionParseException("run command requires a command to run!"); - } - - ZenServerEnvironment TestEnv; - std::filesystem::path ProgramBaseDir = std::filesystem::path(argv[0]).parent_path(); - std::filesystem::path TestBaseDir = ProgramBaseDir.parent_path().parent_path() / ".test"; - TestEnv.InitializeForTest(ProgramBaseDir, TestBaseDir); - - std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); - - ZenServerInstance Zen1(TestEnv); - Zen1.SetTestDir(TestDir); - Zen1.SpawnServer(13337); - - if (!ParseOptions(argc, argv)) - { - return 0; - } - auto result = m_Options.parse(argc, argv); - - std::filesystem::path TreePath{m_ExeTree}; - - struct Visitor : public zen::FileSystemTraversal::TreeVisitor - { - const std::filesystem::path& m_RootPath; - - Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {} - - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override - { - std::filesystem::path FullPath = Parent / FileName; - - zen::IoHashStream Ios; - zen::ScanFile(FullPath, 64 * 1024, [&](const void* Data, size_t Size) { Ios.Append(Data, Size); }); - zen::IoHash Hash = Ios.GetHash(); - - path_string RelativePath = FullPath.lexically_relative(m_RootPath).native(); - // ZEN_INFO("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); - - FileEntry& Entry = m_Files[RelativePath]; - Entry.Hash = Hash; - Entry.Size = FileSize; - - m_HashToFile[Hash] = FullPath; - } - - virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - std::filesystem::path FullPath = Parent / DirectoryName; - - if (DirectoryName.starts_with('.')) - { - return false; - } - - return true; - } - - struct FileEntry - { - uint64_t Size; - zen::IoHash Hash; - }; - - std::map<path_string, FileEntry> m_Files; - std::unordered_map<zen::IoHash, std::filesystem::path, zen::IoHash::Hasher> m_HashToFile; - }; - - zen::FileSystemTraversal Traversal; - Visitor Visit(TreePath); - Traversal.TraverseFileSystem(TreePath, Visit); - - zen::CbObjectWriter PrepReq; - PrepReq << "cmd" << GlobalOptions.PassthroughV[0]; - PrepReq << "args" << GlobalOptions.PassthroughArgs; - PrepReq.BeginArray("files"); - - for (const auto& Kv : Visit.m_Files) - { - PrepReq.BeginObject(); - PrepReq << "file" << zen::PathToUtf8(Kv.first) << "size" << Kv.second.Size << "hash" << Kv.second.Hash; - PrepReq.EndObject(); - } - PrepReq.EndArray(); - - zen::BinaryWriter MemOut; - PrepReq.Save(MemOut); - - Zen1.WaitUntilReady(); - - cpr::Response Response = - cpr::Post(cpr::Url("http://localhost:13337/exec/jobs/prep"), cpr::Body((const char*)MemOut.Data(), MemOut.Size())); - - if (Response.status_code < 300) - { - zen::IoBuffer Payload(zen::IoBuffer::Clone, Response.text.data(), Response.text.size()); - zen::CbObject Result = zen::LoadCompactBinaryObject(Payload); - - for (auto& Need : Result["need"]) - { - zen::IoHash NeedHash = Need.AsHash(); - - if (auto It = Visit.m_HashToFile.find(NeedHash); It != Visit.m_HashToFile.end()) - { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); - - cpr::Response CasResponse = - cpr::Post(cpr::Url("http://localhost:13337/cid"), cpr::Body((const char*)FileData.Data(), FileData.Size())); - - if (CasResponse.status_code >= 300) - { - ZEN_ERROR("CAS put failed with {}", CasResponse.status_code); - } - } - else - { - ZEN_ERROR("unknown hash in 'need' list: {}", NeedHash); - } - } - } - - cpr::Response JobResponse = - cpr::Post(cpr::Url("http://localhost:13337/exec/jobs"), cpr::Body((const char*)MemOut.Data(), MemOut.Size())); - - ZEN_CONSOLE("job exec: {}", JobResponse.status_code); - - return 0; -} - -} // namespace zen - -#endif // ZEN_WITH_EXEC_SERVICES diff --git a/zen/cmds/run.h b/zen/cmds/run.h deleted file mode 100644 index 440ea4ea4..000000000 --- a/zen/cmds/run.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "../zen.h" - -#if ZEN_WITH_EXEC_SERVICES - -namespace zen { - -/** Execute a command (using Zen) - */ -class RunCommand : public ZenCmdBase -{ -public: - RunCommand(); - ~RunCommand(); - - virtual int Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; - virtual cxxopts::Options& Options() override { return m_Options; } - -private: - cxxopts::Options m_Options{"run", "Run command"}; - std::string m_TargetHost; - std::string m_ExeTree; -}; - -} // namespace zen - -#endif // ZEN_WITH_EXEC_SERVICES diff --git a/zen/xmake.lua b/zen/xmake.lua index c4ef713bb..86d95406e 100644 --- a/zen/xmake.lua +++ b/zen/xmake.lua @@ -13,12 +13,12 @@ target("zen") end if is_plat("windows") then + add_files("zen.rc") add_ldflags("/subsystem:console,5.02") add_ldflags("/LTCG") - add_files("zen.rc") + add_ldflags("crypt32.lib", "wldap32.lib", "Ws2_32.lib") end - if is_plat("macosx") then add_ldflags("-framework CoreFoundation") add_ldflags("-framework Security") diff --git a/zen/zen.cpp b/zen/zen.cpp index b4159df3a..e02f3a510 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -14,7 +14,6 @@ #include "cmds/importproject.h" #include "cmds/print.h" #include "cmds/projectstore.h" -#include "cmds/run.h" #include "cmds/scrub.h" #include "cmds/status.h" #include "cmds/top.h" @@ -144,13 +143,10 @@ main(int argc, char** argv) auto _ = zen::MakeGuard([] { spdlog::shutdown(); }); - HashCommand HashCmd; - CopyCommand CopyCmd; - DedupCommand DedupCmd; - DropCommand DropCmd; -#if ZEN_WITH_EXEC_SERVICES - RunCommand RunCmd; -#endif + HashCommand HashCmd; + CopyCommand CopyCmd; + DedupCommand DedupCmd; + DropCommand DropCmd; StatusCommand StatusCmd; TopCommand TopCmd; PrintCommand PrintCmd; @@ -187,9 +183,6 @@ main(int argc, char** argv) {"import-project", &ImportProjectCmd, "Import project store oplog"}, {"print", &PrintCmd, "Print compact binary object"}, {"printpackage", &PrintPkgCmd, "Print compact binary package"}, -#if ZEN_WITH_EXEC_SERVICES - {"run", &RunCmd, "Remote execution"}, -#endif // ZEN_WITH_EXEC_SERVICES {"status", &StatusCmd, "Show zen status"}, {"ps", &PsCmd, "Enumerate running zen server instances"}, {"top", &TopCmd, "Monitor zen server activity"}, diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index ee3c5b742..6117abea9 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -2317,181 +2317,6 @@ TEST_CASE("zcache.rpc.allpolicies") } } -# if ZEN_WITH_EXEC_SERVICES - -struct RemoteExecutionRequest -{ - RemoteExecutionRequest(std::string_view Host, int Port, std::filesystem::path& TreePath) - : m_HostName(Host) - , m_PortNumber(Port) - , m_TreePath(TreePath) - { - } - - void Build(std::string_view Command, std::string_view Arguments) - { - zen::FileSystemTraversal Traversal; - Traversal.TraverseFileSystem(m_TreePath, m_Visit); - - zen::CbObjectWriter PrepReq; - - PrepReq << "cmd" << Command; - PrepReq << "args" << Arguments; - - PrepReq.BeginArray("files"); - - for (const auto& Kv : m_Visit.m_Files) - { - PrepReq.BeginObject(); - PrepReq << "file" << zen::PathToUtf8(Kv.first) << "size" << Kv.second.Size << "hash" << Kv.second.Hash; - PrepReq.EndObject(); - } - PrepReq.EndArray(); - - PrepReq.Save(m_MemOut); - } - - void Prep() - { - cpr::Response Response = - cpr::Post(cpr::Url(fmt::format("{}/prep", m_BaseUri)), cpr::Body((const char*)m_MemOut.Data(), m_MemOut.Size())); - - if (Response.status_code < 300) - { - zen::IoBuffer Payload(zen::IoBuffer::Clone, Response.text.data(), Response.text.size()); - zen::CbObject Result = zen::LoadCompactBinaryObject(Payload); - - for (auto& Need : Result["need"]) - { - zen::IoHash NeedHash = Need.AsHash(); - - if (auto It = m_Visit.m_HashToFile.find(NeedHash); It != m_Visit.m_HashToFile.end()) - { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); - - cpr::Response CidResponse = cpr::Post(cpr::Url(m_CidUri), cpr::Body((const char*)FileData.Data(), FileData.Size())); - - if (CidResponse.status_code >= 300) - { - ZEN_ERROR("CID put failed with {}", CidResponse.status_code); - } - } - else - { - ZEN_ERROR("unknown hash in 'need' list: {}", NeedHash); - } - } - } - } - - zen::CbObject Exec() - { - cpr::Response JobResponse = cpr::Post(cpr::Url(m_BaseUri), cpr::Body((const char*)m_MemOut.Data(), m_MemOut.Size())); - - if (JobResponse.status_code < 300) - { - zen::IoBuffer Payload(zen::IoBuffer::Clone, JobResponse.text.data(), JobResponse.text.size()); - return zen::LoadCompactBinaryObject(std::move(Payload)); - } - - ZEN_INFO("job exec: {}", JobResponse.status_code); - return {}; - } - -private: - struct Visitor : public zen::FileSystemTraversal::TreeVisitor - { - const std::filesystem::path& m_RootPath; - - Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {} - - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override - { - std::filesystem::path FullPath = Parent / FileName; - - zen::IoHashStream Ios; - zen::ScanFile(FullPath, 64 * 1024, [&](const void* Data, size_t Size) { Ios.Append(Data, Size); }); - zen::IoHash Hash = Ios.GetHash(); - - auto RelativePath = FullPath.lexically_relative(m_RootPath).native(); - // ZEN_INFO("File: {:32} => {} ({})", zen::PathToUtf8(RelativePath), Hash, FileSize); - - FileEntry& Entry = m_Files[RelativePath]; - Entry.Hash = Hash; - Entry.Size = FileSize; - - m_HashToFile[Hash] = FullPath; - } - - virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) override - { - std::filesystem::path FullPath = Parent / DirectoryName; - - if (DirectoryName.starts_with('.')) - { - return false; - } - - return true; - } - - struct FileEntry - { - uint64_t Size; - zen::IoHash Hash; - }; - - std::map<std::filesystem::path::string_type, FileEntry> m_Files; - std::unordered_map<zen::IoHash, std::filesystem::path, zen::IoHash::Hasher> m_HashToFile; - }; - - std::string m_HostName; - int m_PortNumber; - std::filesystem::path m_TreePath; - const std::string m_BaseUri = fmt::format("http://{}:{}/exec/jobs", m_HostName, m_PortNumber); - const std::string m_CidUri = fmt::format("http://{}:{}/cid", m_HostName, m_PortNumber); - Visitor m_Visit{m_TreePath}; - zen::BinaryWriter m_MemOut; -}; - -TEST_CASE(".exec.basic") -{ - if (true) - { - return; - } - using namespace std::literals; - - std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); - - const uint16_t PortNumber = 13337; - - ZenServerInstance Zen1(TestEnv); - Zen1.SetTestDir(TestDir); - Zen1.SpawnServer(PortNumber); - Zen1.WaitUntilReady(); - - std::filesystem::path TreePath = TestEnv.GetTestRootDir("test/remote1"); - - { - RemoteExecutionRequest RemoteRequest("localhost", PortNumber, TreePath); - RemoteRequest.Build("zentest-appstub.exe", ""); - RemoteRequest.Prep(); - zen::CbObject Result = RemoteRequest.Exec(); - - CHECK(Result["exitcode"sv].AsInt32(-1) == 0); - } - - { - RemoteExecutionRequest RemoteRequest("localhost", PortNumber, TreePath); - RemoteRequest.Build("zentest-appstub.exe", "-f=1"); - RemoteRequest.Prep(); - zen::CbObject Result = RemoteRequest.Exec(); - CHECK(Result["exitcode"sv].AsInt32(-1) == 1); - } -} -# endif // ZEN_WITH_EXEC_SERVICES - TEST_CASE("mesh.basic") { // --mesh option only available with ZEN_ENABLE_MESH diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp deleted file mode 100644 index b26f9e437..000000000 --- a/zenserver/testing/launch.cpp +++ /dev/null @@ -1,565 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "launch.h" - -#if ZEN_WITH_EXEC_SERVICES - -# include <zencore/compactbinary.h> -# include <zencore/compactbinarybuilder.h> -# include <zencore/compress.h> -# include <zencore/filesystem.h> -# include <zencore/fmtutils.h> -# include <zencore/iobuffer.h> -# include <zencore/iohash.h> -# include <zencore/logging.h> -# include <zencore/windows.h> -# include <zenstore/cidstore.h> - -ZEN_THIRD_PARTY_INCLUDES_START -# include <AccCtrl.h> -# include <AclAPI.h> -# include <UserEnv.h> -# include <atlbase.h> -# include <sddl.h> -ZEN_THIRD_PARTY_INCLUDES_END -# pragma comment(lib, "UserEnv.lib") - -# include <filesystem> -# include <span> - -using namespace std::literals; - -namespace zen { - -struct BasicJob -{ -public: - BasicJob() = default; - ~BasicJob(); - - void SetWorkingDirectory(const std::filesystem::path& WorkingDirectory) { m_WorkingDirectory = WorkingDirectory; } - bool SpawnJob(std::filesystem::path ExePath, std::wstring CommandLine); - bool Wait(uint32_t TimeoutMs = ~0); - int ExitCode(); - -private: - std::filesystem::path m_WorkingDirectory; - int m_ProcessId = 0; - CHandle m_ProcessHandle; -}; - -BasicJob::~BasicJob() -{ - Wait(); -} - -bool -BasicJob::SpawnJob(std::filesystem::path ExePath, std::wstring CommandLine) -{ - STARTUPINFOEX StartupInfo = {sizeof(STARTUPINFOEX)}; - PROCESS_INFORMATION ProcessInfo{}; - - std::wstring ExePathNative = ExePath.native(); - std::wstring WorkingDirNative = m_WorkingDirectory.native(); - - BOOL Created = ::CreateProcess(ExePathNative.data() /* ApplicationName */, - CommandLine.data() /* Command Line */, - nullptr /* Process Attributes */, - nullptr /* Security Attributes */, - FALSE /* InheritHandles */, - 0 /* Flags */, - nullptr /* Environment */, - WorkingDirNative.data() /* Current Directory */, - (LPSTARTUPINFO)&StartupInfo, - &ProcessInfo); - - if (!Created) - { - throw std::system_error(::GetLastError(), std::system_category(), fmt::format("Failed to create process '{}'", ExePath).c_str()); - } - - m_ProcessId = ProcessInfo.dwProcessId; - m_ProcessHandle.Attach(ProcessInfo.hProcess); - ::CloseHandle(ProcessInfo.hThread); - - ZEN_INFO("Created process {}", m_ProcessId); - - return true; -} - -bool -BasicJob::Wait(uint32_t TimeoutMs) -{ - if (!m_ProcessHandle) - { - return true; - } - - DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, TimeoutMs); - - if (WaitResult == WAIT_TIMEOUT) - { - return false; - } - - if (WaitResult == WAIT_OBJECT_0) - { - return true; - } - - throw std::runtime_error("Failed wait on process handle"); -} - -int -BasicJob::ExitCode() -{ - DWORD Ec = 0; - BOOL Success = GetExitCodeProcess(m_ProcessHandle, &Ec); - - if (!Success) - { - ZEN_WARN("failed getting exit code"); - } - - if (Ec == STILL_ACTIVE) - { - ZEN_WARN("getting exit code but process is STILL_ACTIVE"); - } - - return gsl::narrow_cast<int>(Ec); -} - -//////////////////////////////////////////////////////////////////////////////// - -struct SandboxedJob -{ - SandboxedJob() = default; - ~SandboxedJob() = default; - - void SetWorkingDirectory(const std::filesystem::path& WorkingDirectory) { m_WorkingDirectory = WorkingDirectory; } - void Initialize(std::string_view AppContainerId); - bool SpawnJob(std::filesystem::path ExePath); - void AddWhitelistFile(const std::filesystem::path& FilePath) { m_WhitelistFiles.push_back(FilePath); } - -private: - bool GrantNamedObjectAccess(PWSTR Name, SE_OBJECT_TYPE Type, ACCESS_MASK AccessMask, bool Recursive); - - std::filesystem::path m_WorkingDirectory; - std::vector<std::filesystem::path> m_WhitelistFiles; - std::vector<std::wstring> m_WhitelistRegistryKeys; - PSID m_AppContainerSid = nullptr; - bool m_IsInitialized = false; -}; - -bool -SandboxedJob::GrantNamedObjectAccess(PWSTR ObjectName, SE_OBJECT_TYPE ObjectType, ACCESS_MASK AccessMask, bool Recursive) -{ - DWORD Status; - PACL NewAcl = nullptr; - - DWORD grfInhericance = 0; - - if (Recursive) - { - grfInhericance = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE; - } - - EXPLICIT_ACCESS Access{.grfAccessPermissions = AccessMask, - .grfAccessMode = GRANT_ACCESS, - .grfInheritance = grfInhericance, - .Trustee = {.pMultipleTrustee = nullptr, - .MultipleTrusteeOperation = NO_MULTIPLE_TRUSTEE, - .TrusteeForm = TRUSTEE_IS_SID, - .TrusteeType = TRUSTEE_IS_GROUP, - .ptstrName = (PWSTR)m_AppContainerSid}}; - - PACL OldAcl = nullptr; - - Status = GetNamedSecurityInfo(ObjectName /* ObjectName */, - ObjectType /* ObjectType */, - DACL_SECURITY_INFORMATION /* SecurityInfo */, - nullptr /* ppsidOwner */, - nullptr /* ppsidGroup */, - &OldAcl /* ppDacl */, - nullptr /* ppSacl */, - nullptr /* ppSecurityDescriptor */); - if (Status != ERROR_SUCCESS) - return false; - - Status = SetEntriesInAcl(1 /* CountOfExplicitEntries */, &Access /* pListOfExplicitEntries */, OldAcl, &NewAcl); - if (Status != ERROR_SUCCESS) - return false; - - Status = SetNamedSecurityInfo(ObjectName /* ObjectName */, - ObjectType /* ObjectType */, - DACL_SECURITY_INFORMATION /*SecurityInfo */, - nullptr /* psidOwner */, - nullptr /* psidGroup */, - NewAcl /* pDacl */, - nullptr /* pSacl */); - if (NewAcl) - ::LocalFree(NewAcl); - - return Status == ERROR_SUCCESS; -} - -void -SandboxedJob::Initialize(std::string_view AppContainerId) -{ - if (m_IsInitialized) - { - return; - } - - std::wstring ContainerName = zen::Utf8ToWide(AppContainerId); - - HRESULT hRes = ::CreateAppContainerProfile(ContainerName.c_str(), - ContainerName.c_str() /* Display Name */, - ContainerName.c_str() /* Description */, - nullptr /* Capabilities */, - 0 /* Capability Count */, - &m_AppContainerSid); - - if (FAILED(hRes)) - { - hRes = ::DeriveAppContainerSidFromAppContainerName(ContainerName.c_str(), &m_AppContainerSid); - - if (FAILED(hRes)) - { - ZEN_ERROR("Failed creating app container SID"); - } - } - - // Debugging context - - PWSTR Str = nullptr; - ::ConvertSidToStringSid(m_AppContainerSid, &Str); - - ZEN_INFO("AppContainer SID : '{}'", WideToUtf8(Str)); - - PWSTR Path = nullptr; - if (SUCCEEDED(::GetAppContainerFolderPath(Str, &Path))) - { - ZEN_INFO("AppContainer folder: '{}'", WideToUtf8(Path)); - - ::CoTaskMemFree(Path); - } - ::LocalFree(Str); - - m_IsInitialized = true; -} - -bool -SandboxedJob::SpawnJob(std::filesystem::path ExePath) -{ - // Build process attributes - - SECURITY_CAPABILITIES Sc = {0}; - Sc.AppContainerSid = m_AppContainerSid; - - STARTUPINFOEX StartupInfo = {sizeof(STARTUPINFOEX)}; - PROCESS_INFORMATION ProcessInfo{}; - SIZE_T Size = 0; - - ::InitializeProcThreadAttributeList(nullptr, 1, 0, &Size); - - auto AttrBuffer = std::make_unique<uint8_t[]>(Size); - StartupInfo.lpAttributeList = reinterpret_cast<LPPROC_THREAD_ATTRIBUTE_LIST>(AttrBuffer.get()); - - if (!::InitializeProcThreadAttributeList(StartupInfo.lpAttributeList, 1, 0, &Size)) - { - return false; - } - - if (!::UpdateProcThreadAttribute(StartupInfo.lpAttributeList, - 0, - PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, - &Sc, - sizeof Sc, - nullptr, - nullptr)) - { - return false; - } - - // Set up security for files/folders/registry - - for (const std::filesystem::path& File : m_WhitelistFiles) - { - std::wstring NativeFileName = File.native(); - GrantNamedObjectAccess(NativeFileName.data(), SE_FILE_OBJECT, FILE_ALL_ACCESS, true); - } - - for (std::wstring& RegKey : m_WhitelistRegistryKeys) - { - GrantNamedObjectAccess(RegKey.data(), SE_REGISTRY_WOW64_32KEY, KEY_ALL_ACCESS, true); - } - - std::wstring ExePathNative = ExePath.native(); - std::wstring WorkingDirNative = m_WorkingDirectory.native(); - - BOOL Created = ::CreateProcess(nullptr /* ApplicationName */, - ExePathNative.data() /* Command line */, - nullptr /* Process Attributes */, - nullptr /* Security Attributes */, - FALSE /* InheritHandles */, - EXTENDED_STARTUPINFO_PRESENT | CREATE_NEW_CONSOLE /* Flags */, - nullptr /* Environment */, - WorkingDirNative.data() /* Current Directory */, - (LPSTARTUPINFO)&StartupInfo, - &ProcessInfo); - - DeleteProcThreadAttributeList(StartupInfo.lpAttributeList); - - if (!Created) - { - return false; - } - - ZEN_INFO("Created process {}", ProcessInfo.dwProcessId); - - return true; -} - -//////////////////////////////////////////////////////////////////////////////// - -HttpLaunchService::HttpLaunchService(CidStore& Store, const std::filesystem::path& SandboxBaseDir) -: m_Log(logging::Get("exec")) -, m_CidStore(Store) -, m_SandboxPath(SandboxBaseDir) -{ - m_Router.AddPattern("job", "([[:digit:]]+)"); - - m_Router.RegisterRoute( - "jobs/{job}", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - break; - - case HttpVerb::kPost: - break; - - default: - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); - - // Experimental - -# if 0 - m_Router.RegisterRoute( - "jobs/sandbox", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - break; - - case HttpVerb::kPost: - { - SandboxedJob Job; - Job.Initialize("zen_test"); - Job.SetWorkingDirectory("c:\\temp\\sandbox1"); - Job.AddWhitelistFile("c:\\temp\\sandbox1"); - Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); - } - break; - - default: - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); -# endif - - m_Router.RegisterRoute( - "jobs/prep", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kPost: - { - // This operation takes the proposed job spec and identifies which - // chunks are not present on this server. This list is then returned in - // the "need" list in the response - - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject RequestObject = LoadCompactBinaryObject(Payload); - - std::vector<IoHash> NeedList; - - for (auto Entry : RequestObject["files"sv]) - { - CbObjectView Ob = Entry.AsObjectView(); - - const IoHash FileHash = Ob["hash"sv].AsHash(); - - if (!m_CidStore.FindChunkByCid(FileHash)) - { - ZEN_DEBUG("NEED: {} {} {}", FileHash, Ob["file"sv].AsString(), Ob["size"sv].AsUInt64()); - - NeedList.push_back(FileHash); - } - } - - CbObjectWriter Cbo; - Cbo.BeginArray("need"); - - for (const IoHash& Hash : NeedList) - { - Cbo << Hash; - } - - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response); - } - break; - - default: - break; - } - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "jobs", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - break; - - case HttpVerb::kPost: - { - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject RequestObject = LoadCompactBinaryObject(Payload); - - bool AllOk = true; - - std::vector<IoHash> NeedList; - - std::filesystem::path SandboxDir{CreateNewSandbox()}; - - ZEN_DEBUG("setting up job in sandbox '{}'", SandboxDir); - - zen::DeleteDirectories(SandboxDir); - zen::CreateDirectories(SandboxDir); - - for (auto Entry : RequestObject["files"sv]) - { - CbObjectView Ob = Entry.AsObjectView(); - - std::string_view FileName = Ob["file"sv].AsString(); - const IoHash FileHash = Ob["hash"sv].AsHash(); - uint64_t FileSize = Ob["size"sv].AsUInt64(); - - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(FileHash); !Chunk) - { - ZEN_DEBUG("MISSING: {} {} {}", FileHash, FileName, FileSize); - AllOk = false; - - NeedList.push_back(FileHash); - } - else - { - std::filesystem::path FullPath = SandboxDir / FileName; - - IoHash RawHash; - uint64_t RawSize; - CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Chunk), RawHash, RawSize); - ZEN_ASSERT(Compressed); - ZEN_ASSERT(FileHash == RawHash); - CompositeBuffer CompositeBuffer = Compressed.DecompressToComposite(); - std::span<const SharedBuffer> Segments = CompositeBuffer.GetSegments(); - std::vector<IoBuffer> Chunks(Segments.size()); - std::vector<IoBuffer*> ChunkPtrs(Segments.size()); - for (size_t Index = 0; Index < Segments.size(); ++Index) - { - Chunks[Index] = std::move(Segments[Index].AsIoBuffer()); - ChunkPtrs[Index] = &Chunks[Index]; - } - - zen::WriteFile(FullPath, ChunkPtrs.data(), ChunkPtrs.size()); - } - } - - if (!AllOk) - { - // TODO: Could report all the missing pieces in the response here - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - std::string Executable8{RequestObject["cmd"].AsString()}; - std::string Args8{RequestObject["args"].AsString()}; - - std::wstring Executable = Utf8ToWide(Executable8); - std::wstring Args = Utf8ToWide(Args8); - - ZEN_DEBUG("spawning job in sandbox '{}': '{}' '{}'", SandboxDir, Executable8, Args8); - - std::filesystem::path ExeName = SandboxDir / Executable; - - BasicJob Job; - Job.SetWorkingDirectory(SandboxDir); - Job.SpawnJob(ExeName, Args); - Job.Wait(); - - CbObjectWriter Response; - - Response << "exitcode" << Job.ExitCode(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); - } - break; - - default: - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); -} - -HttpLaunchService::~HttpLaunchService() -{ -} - -const char* -HttpLaunchService::BaseUri() const -{ - return "/exec/"; -} - -void -HttpLaunchService::HandleRequest(HttpServerRequest& Request) -{ - if (m_Router.HandleRequest(Request) == false) - { - ZEN_WARN("No route found for {0}", Request.RelativeUri()); - } -} - -std::filesystem::path -HttpLaunchService::CreateNewSandbox() -{ - std::string UniqueId = std::to_string(++m_SandboxCount); - std::filesystem::path Path = m_SandboxPath / UniqueId; - zen::CreateDirectories(Path); - return Path; -} - -} // namespace zen - -#endif // ZEN_WITH_EXEC_SERVICES diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h deleted file mode 100644 index f44618bfb..000000000 --- a/zenserver/testing/launch.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include <zencore/zencore.h> - -#if !defined(ZEN_WITH_EXEC_SERVICES) -# define ZEN_WITH_EXEC_SERVICES ZEN_PLATFORM_WINDOWS -#endif - -#if ZEN_WITH_EXEC_SERVICES - -# include <zencore/logging.h> -# include <zenhttp/httpserver.h> - -# include <filesystem> - -namespace zen { - -class CidStore; - -/** - * Process launcher for test executables - */ -class HttpLaunchService : public HttpService -{ -public: - HttpLaunchService(CidStore& Store, const std::filesystem::path& SandboxBaseDir); - ~HttpLaunchService(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; - -private: - inline spdlog::logger& Log() { return m_Log; } - - spdlog::logger& m_Log; - HttpRequestRouter m_Router; - CidStore& m_CidStore; - std::filesystem::path m_SandboxPath; - std::atomic<int> m_SandboxCount{0}; - - std::filesystem::path CreateNewSandbox(); -}; - -} // namespace zen - -#endif // ZEN_WITH_EXEC_SERVICES diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9ca71ed81..606047105 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -110,7 +110,6 @@ ZEN_THIRD_PARTY_INCLUDES_END #include "monitoring/httpstatus.h" #include "projectstore.h" #include "testing/httptest.h" -#include "testing/launch.h" #include "upstream/upstream.h" #include "zenstore/gc.h" @@ -265,23 +264,6 @@ public: m_ProjectStore = new zen::ProjectStore(*m_CidStore, m_DataRoot / "projects", m_GcManager); m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore}); -#if ZEN_WITH_EXEC_SERVICES - - if (ServerOptions.ExecServiceEnabled) - { - ZEN_INFO("instantiating exec service"); - - std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; - zen::CreateDirectories(SandboxDir); - m_HttpLaunchService = std::make_unique<zen::HttpLaunchService>(*m_CidStore, SandboxDir); - } - else - { - ZEN_INFO("NOT instantiating exec services"); - } - -#endif // ZEN_WITH_EXEC_SERVICES - #if ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.ComputeServiceEnabled) { @@ -329,15 +311,6 @@ public: m_Http->RegisterService(*m_CidService); -#if ZEN_WITH_EXEC_SERVICES - if (ServerOptions.ExecServiceEnabled) - { - if (m_HttpLaunchService != nullptr) - { - m_Http->RegisterService(*m_HttpLaunchService); - } - } -#endif // ZEN_WITH_EXEC_SERVICES #if ZEN_WITH_COMPUTE_SERVICES if (ServerOptions.ComputeServiceEnabled) { @@ -611,9 +584,6 @@ private: zen::HttpAdminService m_AdminService{m_GcScheduler}; zen::HttpHealthService m_HealthService; zen::MeshTracker m_ZenMesh{m_IoContext}; -#if ZEN_WITH_EXEC_SERVICES - std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; -#endif // ZEN_WITH_EXEC_SERVICES #if ZEN_WITH_COMPUTE_SERVICES std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService; #endif // ZEN_WITH_COMPUTE_SERVICES |