diff options
| author | Martin Ridgers <[email protected]> | 2021-12-08 09:04:04 +0100 |
|---|---|---|
| committer | Martin Ridgers <[email protected]> | 2021-12-08 09:04:04 +0100 |
| commit | 1037aa9edc9b88d9c59fdf2d2531ac265b571b3c (patch) | |
| tree | 805f277f60b1eba7de8b78ae209d7464bcedd398 | |
| parent | Implement zen/internalfile for POSIX platforms (diff) | |
| parent | Use 'Platform' instead of 'OSFamily' for Horde condition (diff) | |
| download | zen-1037aa9edc9b88d9c59fdf2d2531ac265b571b3c.tar.xz zen-1037aa9edc9b88d9c59fdf2d2531ac265b571b3c.zip | |
Merged main
| -rw-r--r-- | zenhttp/httpasio.cpp | 38 | ||||
| -rw-r--r-- | zenhttp/httpsys.cpp | 88 | ||||
| -rw-r--r-- | zenhttp/httpsys.h | 20 | ||||
| -rw-r--r-- | zenserver/compute/apply.cpp | 31 | ||||
| -rw-r--r-- | zenserver/compute/apply.h | 2 | ||||
| -rw-r--r-- | zenserver/config.cpp | 33 | ||||
| -rw-r--r-- | zenserver/upstream/upstreamapply.cpp | 28 | ||||
| -rw-r--r-- | zenserver/xmake.lua | 1 | ||||
| -rw-r--r-- | zenserver/zenserver.vcxproj | 2 | ||||
| -rw-r--r-- | zenutil/zenserverprocess.cpp | 24 |
10 files changed, 204 insertions, 63 deletions
diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp index e2b9262ff..131c4513f 100644 --- a/zenhttp/httpasio.cpp +++ b/zenhttp/httpasio.cpp @@ -124,7 +124,7 @@ struct HttpRequest HttpVerb RequestVerb() const { return m_RequestVerb; } bool IsKeepAlive() const { return m_KeepAlive; } - std::string_view Url() const { return std::string_view(m_Url, m_UrlLength); } + std::string_view Url() const { return m_NormalizedUrl.empty() ? std::string_view(m_Url, m_UrlLength) : m_NormalizedUrl; } std::string_view QueryString() const { return std::string_view(m_QueryString, m_QueryLength); } IoBuffer Body() { return m_BodyBuffer; } @@ -181,6 +181,7 @@ private: uint64_t m_BodyPosition = 0; http_parser m_Parser; char m_HeaderBuffer[1024]; + std::string m_NormalizedUrl; void AppendInputBytes(const char* Data, size_t Bytes); void AppendCurrentHeader(); @@ -752,6 +753,37 @@ HttpRequest::TerminateConnection() m_Connection.TerminateConnection(); } +static void +NormalizeUrlPath(const char* Url, size_t UrlLength, std::string& NormalizedUrl) +{ + bool LastCharWasSeparator = false; + for (std::string_view::size_type UrlIndex = 0; UrlIndex < UrlLength; ++UrlIndex) + { + const char UrlChar = Url[UrlIndex]; + const bool IsSeparator = (UrlChar == '/'); + + if (IsSeparator && LastCharWasSeparator) + { + if (NormalizedUrl.empty()) + { + NormalizedUrl.reserve(UrlLength); + NormalizedUrl.append(Url, UrlIndex); + } + + if (!LastCharWasSeparator) + { + NormalizedUrl.push_back('/'); + } + } + else if (!NormalizedUrl.empty()) + { + NormalizedUrl.push_back(UrlChar); + } + + LastCharWasSeparator = IsSeparator; + } +} + int HttpRequest::OnHeadersComplete() { @@ -818,6 +850,9 @@ HttpRequest::OnHeadersComplete() m_QueryLength = Url.size() - QuerySplit - 1; } + NormalizeUrlPath(m_Url, m_UrlLength, m_NormalizedUrl); + + return 0; } @@ -858,6 +893,7 @@ HttpRequest::ResetState() m_BodyBuffer = {}; m_BodyPosition = 0; m_Headers.clear(); + m_NormalizedUrl.clear(); } int diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index fb1c43537..57763e7a1 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -748,15 +748,20 @@ HttpSysServer::~HttpSysServer() } void -HttpSysServer::Initialize(const wchar_t* UrlPath) +HttpSysServer::InitializeServer(int BasePort) { + using namespace std::literals; + + WideStringBuilder<64> WildcardUrlPath; + WildcardUrlPath << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; + m_IsOk = false; ULONG Result = HttpCreateServerSession(HTTPAPI_VERSION_2, &m_HttpSessionId, 0); if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create server session for '{}': {:#x}", WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create server session for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); return; } @@ -765,18 +770,44 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create URL group for '{}': {:#x}", WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create URL group for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); return; } - m_BaseUri = UrlPath; + Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, WildcardUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); - Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, UrlPath, HTTP_URL_CONTEXT(0), 0); + m_BaseUris.clear(); + if (Result == NO_ERROR) + { + m_BaseUris.push_back(WildcardUrlPath.c_str()); + } + else + { + // If we can't register the wildcard path, we fall back to local paths + // This local paths allow requests originating locally to function, but will not allow + // remote origin requests to function. This can be remedied by using netsh + // during an install process to grant permissions to route public access to the appropriate + // port for the current user. eg: + // netsh http add urlacl url=http://*:1337/ user=<some_user> - if (Result != NO_ERROR) + const std::u8string_view Hosts[] = { u8"[::1]"sv, u8"localhost"sv, u8"127.0.0.1"sv }; + + for (const std::u8string_view Host : Hosts) + { + WideStringBuilder<64> LocalUrlPath; + LocalUrlPath << u8"http://"sv << Host << u8":"sv << int64_t(BasePort) << u8"/"sv; + + if (HttpAddUrlToUrlGroup(m_HttpUrlGroupId, LocalUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0) == NO_ERROR) + { + m_BaseUris.push_back(LocalUrlPath.c_str()); + } + } + } + + if (m_BaseUris.empty()) { - ZEN_ERROR("Failed to add base URL to URL group for '{}': {:#x}", WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to add base URL to URL group for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); return; } @@ -791,7 +822,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create request queue for '{}': {:#x}", WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to create request queue for '{}': {:#x}", WideToUtf8(m_BaseUris.front()), Result); return; } @@ -803,7 +834,7 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to set server binding property for '{}': {:#x}", WideToUtf8(UrlPath), Result); + ZEN_ERROR("Failed to set server binding property for '{}': {:#x}", WideToUtf8(m_BaseUris.front()), Result); return; } @@ -815,13 +846,13 @@ HttpSysServer::Initialize(const wchar_t* UrlPath) if (ErrorCode) { - ZEN_ERROR("Failed to create IOCP for '{}': {}", WideToUtf8(UrlPath), ErrorCode.message()); + ZEN_ERROR("Failed to create IOCP for '{}': {}", WideToUtf8(m_BaseUris.front()), ErrorCode.message()); } else { m_IsOk = true; - ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(UrlPath)); + ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(m_BaseUris.front())); } } @@ -952,15 +983,18 @@ HttpSysServer::RegisterService(const char* UrlPath, HttpService& Service) // Convert to wide string - std::wstring Url16 = m_BaseUri + PathUtf16; + for (const std::wstring& BaseUri : m_BaseUris) + { + std::wstring Url16 = BaseUri + PathUtf16; - ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); + ULONG Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, Url16.c_str(), HTTP_URL_CONTEXT(&Service), 0 /* Reserved */); - if (Result != NO_ERROR) - { - ZEN_ERROR("HttpAddUrlToUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + if (Result != NO_ERROR) + { + ZEN_ERROR("HttpAddUrlToUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); - return; + return; + } } } @@ -978,13 +1012,16 @@ HttpSysServer::UnregisterService(const char* UrlPath, HttpService& Service) // Convert to wide string - std::wstring Url16 = m_BaseUri + PathUtf16; + for (const std::wstring& BaseUri : m_BaseUris) + { + std::wstring Url16 = BaseUri + PathUtf16; - ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); + ULONG Result = HttpRemoveUrlFromUrlGroup(m_HttpUrlGroupId, Url16.c_str(), 0); - if (Result != NO_ERROR) - { - ZEN_ERROR("HttpRemoveUrlFromUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + if (Result != NO_ERROR) + { + ZEN_ERROR("HttpRemoveUrlFromUrlGroup failed with result: '{}'", GetSystemErrorAsString(Result)); + } } } @@ -1570,12 +1607,7 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT void HttpSysServer::Initialize(int BasePort) { - using namespace std::literals; - - WideStringBuilder<64> BaseUri; - BaseUri << u8"http://*:"sv << int64_t(BasePort) << u8"/"sv; - - Initialize(BaseUri.c_str()); + InitializeServer(BasePort); StartServer(); } diff --git a/zenhttp/httpsys.h b/zenhttp/httpsys.h index 46ba122cc..7df8fba8f 100644 --- a/zenhttp/httpsys.h +++ b/zenhttp/httpsys.h @@ -52,7 +52,7 @@ public: inline bool IsAsyncResponseEnabled() const { return m_IsAsyncResponseEnabled; } private: - void Initialize(const wchar_t* UrlPath); + void InitializeServer(int BasePort); void Cleanup(); void StartServer(); @@ -75,15 +75,15 @@ private: WinIoThreadPool m_ThreadPool; WorkerThreadPool m_AsyncWorkPool; - std::wstring m_BaseUri; // http://*:nnnn/ - HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; - HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; - HANDLE m_RequestQueueHandle = 0; - std::atomic_int32_t m_PendingRequests{0}; - std::atomic<int32_t> m_IsShuttingDown{0}; - int32_t m_MinPendingRequests = 16; - int32_t m_MaxPendingRequests = 128; - Event m_ShutdownEvent; + std::vector<std::wstring> m_BaseUris; // eg: http://*:nnnn/ + HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; + HTTP_URL_GROUP_ID m_HttpUrlGroupId = 0; + HANDLE m_RequestQueueHandle = 0; + std::atomic_int32_t m_PendingRequests{0}; + std::atomic_int32_t m_IsShuttingDown{0}; + int32_t m_MinPendingRequests = 16; + int32_t m_MaxPendingRequests = 128; + Event m_ShutdownEvent; }; } // namespace zen diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 1db3fe631..ddf7ad36c 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -593,9 +593,13 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, if (NeedList.empty()) { // We already have everything + CbObject Output; + HttpResponseCode ResponseCode = ExecActionUpstream(Worker, RequestObject, Output); - CbObject Output = ExecActionUpstream(Worker, RequestObject); - + if (ResponseCode != HttpResponseCode::OK) + { + return HttpReq.WriteResponse(ResponseCode); + } return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } @@ -655,8 +659,13 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, zen::NiceBytes(TotalNewBytes), NewAttachmentCount); - CbObject Output = ExecActionUpstream(Worker, ActionObj); + CbObject Output; + HttpResponseCode ResponseCode = ExecActionUpstream(Worker, ActionObj, Output); + if (ResponseCode != HttpResponseCode::OK) + { + return HttpReq.WriteResponse(ResponseCode); + } return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; @@ -904,8 +913,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) return OutputPackage; } -CbObject -HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Action) +HttpResponseCode +HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Action, CbObject& Object) { const IoHash WorkerId = Worker.Descriptor.GetHash(); const IoHash ActionId = Action.GetHash(); @@ -918,14 +927,19 @@ HttpFunctionService::ExecActionUpstream(const WorkerDesc& Worker, CbObject Actio if (!EnqueueResult.Success) { - throw std::runtime_error("Error enqueuing upstream task"); + ZEN_ERROR( + "Error enqueuing upstream Action {}/{}", + WorkerId.ToHexString(), + ActionId.ToHexString()); + return HttpResponseCode::InternalServerError; } CbObjectWriter Writer; Writer.AddHash("worker", WorkerId); Writer.AddHash("action", ActionId); - return std::move(Writer.Save()); + Object = std::move(Writer.Save()); + return HttpResponseCode::OK; } HttpResponseCode @@ -955,8 +969,7 @@ HttpFunctionService::ExecActionUpstreamResult(const IoHash& WorkerId, const IoHa Completed.Error.Reason, Completed.Error.ErrorCode); - throw std::runtime_error( - "Action {}/{} failed"_format(WorkerId.ToHexString(), ActionId.ToHexString()).c_str()); + return HttpResponseCode::InternalServerError; } ZEN_INFO("Action {}/{} completed with {} attachments ({} compressed, {} uncompressed)", diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index a3e36819d..af8668ee2 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -54,7 +54,7 @@ private: [[nodiscard]] std::filesystem::path CreateNewSandbox(); [[nodiscard]] CbPackage ExecAction(const WorkerDesc& Worker, CbObject Action); - [[nodiscard]] CbObject ExecActionUpstream(const WorkerDesc& Worker, CbObject Action); + [[nodiscard]] HttpResponseCode ExecActionUpstream(const WorkerDesc& Worker, CbObject Action, CbObject& Object); [[nodiscard]] HttpResponseCode ExecActionUpstreamResult(const IoHash& WorkerId, const IoHash& ActionId, CbPackage& Package); RwLock m_WorkerLock; diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 84eb4ae75..354df7b25 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -47,6 +47,39 @@ PickDefaultStateDirectory() return myDocumentsDir; } + int CandidateDriveLetterOffset = -1; + ULARGE_INTEGER CandidateDriveSize; + CandidateDriveSize.QuadPart = 0L; + DWORD LogicalDrives = GetLogicalDrives(); + char CandidateDriveName[] = "A:\\"; + for (int DriveLetterOffset = 0; DriveLetterOffset < 32; ++DriveLetterOffset) + { + if ((LogicalDrives & (1 << DriveLetterOffset)) != 0) + { + CandidateDriveName[0] = (char)('A' + DriveLetterOffset); + if (GetDriveTypeA(CandidateDriveName) == DRIVE_FIXED) + { + ULARGE_INTEGER FreeBytesAvailableToCaller; + ULARGE_INTEGER TotalNumberOfBytes; + if (0 != GetDiskFreeSpaceExA(CandidateDriveName, &FreeBytesAvailableToCaller, &TotalNumberOfBytes, nullptr)) + { + if ((FreeBytesAvailableToCaller.QuadPart > 0) && (TotalNumberOfBytes.QuadPart > CandidateDriveSize.QuadPart)) + { + CandidateDriveLetterOffset = DriveLetterOffset; + CandidateDriveSize = TotalNumberOfBytes; + } + } + } + } + } + + if (CandidateDriveLetterOffset >= 0) + { + char RootZenFolderName[] = "A:\\zen"; + RootZenFolderName[0] = (char)('A' + CandidateDriveLetterOffset); + return RootZenFolderName; + } + return L""; } diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 6ff5c5da2..2b805fe14 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -40,6 +40,9 @@ namespace zen { using namespace std::literals; +static const IoBuffer EmptyBuffer; +static const IoHash EmptyBufferId = IoHash::HashBuffer(EmptyBuffer); + namespace detail { class HordeUpstreamApplyEndpoint final : public UpstreamApplyEndpoint @@ -708,6 +711,8 @@ namespace detail { [[nodiscard]] bool ProcessApplyKey(const UpstreamApplyRecord& ApplyRecord, UpstreamData& Data) { + using namespace fmt::literals; + std::string ExecutablePath; std::map<std::string, std::string> Environment; std::set<std::filesystem::path> InputFiles; @@ -739,6 +744,15 @@ namespace detail { } } + for (auto& It : ApplyRecord.WorkerDescriptor["dirs"sv]) + { + std::string_view Directory = It.AsString(); + std::string DummyFile = "{}/.zen_empty_file"_format(Directory); + InputFiles.insert(DummyFile); + Data.Blobs[EmptyBufferId] = EmptyBuffer; + InputFileHashes[DummyFile] = EmptyBufferId; + } + for (auto& It : ApplyRecord.WorkerDescriptor["environment"sv]) { std::string_view Env = It.AsString(); @@ -813,18 +827,10 @@ namespace detail { bool Exclusive = ApplyRecord.WorkerDescriptor["exclusive"sv].AsBool(); // TODO: Remove override when Horde accepts the UE style Host Platforms (Win64, Linux, Mac) - std::string Condition; - if (HostPlatform == "Win64" || HostPlatform == "Windows") - { - Condition = "OSFamily == 'Windows' && Pool == 'Win-RemoteExec'"; - } - else if (HostPlatform == "Mac") - { - Condition = "OSFamily == 'MacOS'"; - } - else + std::string Condition = "Platform == '{}'"_format(HostPlatform); + if (HostPlatform == "Win64") { - Condition = "OSFamily == '{}'"_format(HostPlatform); + Condition += " && Pool == 'Win-RemoteExec'"; } std::map<std::string_view, int64_t> Resources; diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index 35094c457..d53fdd4f0 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -12,7 +12,6 @@ target("zenserver") if is_plat("windows") then add_ldflags("/subsystem:console,5.02") add_ldflags("/MANIFEST:EMBED") - add_ldflags("/MANIFESTUAC:level='requireAdministrator'") add_ldflags("/LTCG") else del_files("windows/**") diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index 935979cc3..056d431cb 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -82,7 +82,6 @@ <EnableCOMDATFolding>true</EnableCOMDATFolding> <OptimizeReferences>true</OptimizeReferences> <GenerateDebugInformation>true</GenerateDebugInformation> - <UACExecutionLevel>RequireAdministrator</UACExecutionLevel> <DelayLoadDLLs>projectedfslib.dll;shell32.dll</DelayLoadDLLs> </Link> </ItemDefinitionGroup> @@ -99,7 +98,6 @@ <Link> <SubSystem>Console</SubSystem> <GenerateDebugInformation>true</GenerateDebugInformation> - <UACExecutionLevel>RequireAdministrator</UACExecutionLevel> <DelayLoadDLLs>projectedfslib.dll;shell32.dll</DelayLoadDLLs> </Link> </ItemDefinitionGroup> diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index 417bd74ab..74fbaf180 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -110,10 +110,19 @@ ZenServerState::Initialize() #if ZEN_PLATFORM_WINDOWS // TODO: there's a small chance of a race here, this logic could be tightened up with a mutex to // ensure only a single process at a time creates the mapping + // TODO: the fallback to Local instead of Global has a flaw where if you start a non-elevated instance + // first then start an elevated instance second you'll have the first instance with a local + // mapping and the second instance with a global mapping. This kind of elevated/non-elevated + // shouldn't be common, but handling for it should be improved in the future. HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); if (hMap == NULL) { + hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap"); + } + + if (hMap == NULL) + { // Security attributes to enable any user to access state zenutil::AnyUserSecurityAttributes Attrs; @@ -126,6 +135,16 @@ ZenServerState::Initialize() if (hMap == NULL) { + hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file + Attrs.Attributes(), // allow anyone to access + PAGE_READWRITE, // read/write access + 0, // maximum object size (high-order DWORD) + m_MaxEntryCount * sizeof(ZenServerEntry), // maximum object size (low-order DWORD) + L"Local\\ZenMap"); // name of mapping object + } + + if (hMap == NULL) + { ThrowLastError("Could not open or create file mapping object for Zen server state"); } } @@ -171,6 +190,11 @@ ZenServerState::InitializeReadOnly() HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); if (hMap == NULL) { + hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Local\\ZenMap"); + } + + if (hMap == NULL) + { return false; } |