From c55df8f9cdded0da0dc6a4d6a42e043e46e9b27f Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 12:51:01 +0200 Subject: GetWindowsErrorAsString() -> GetErrorAsString() --- zencore/except.cpp | 6 +++--- zencore/include/zencore/except.h | 2 +- zenhttp/httpsys.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zencore/except.cpp b/zencore/except.cpp index d57b539cd..123d8e231 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -31,13 +31,13 @@ ThrowLastError(std::string_view Message) std::string GetLastErrorAsString() { - return GetWindowsErrorAsString(zen::GetLastError()); + return GetErrorAsString(zen::GetLastError()); } std::string -GetWindowsErrorAsString(uint32_t Win32ErrorCode) +GetErrorAsString(uint32_t ErrorCode) { - return std::error_code(Win32ErrorCode, std::system_category()).message(); + return std::error_code(ErrorCode, std::system_category()).message(); } void diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 486f3e380..5e0f580f2 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -56,7 +56,7 @@ ZENCORE_API void ThrowSystemException(HRESULT hRes, std::string_view Message); ZENCORE_API void ThrowLastError(std::string_view Message); ZENCORE_API void ThrowLastError(std::string_view Message, const std::source_location& Location); ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); +ZENCORE_API std::string GetErrorAsString(uint32_t ErrorCode); inline void ThrowSystemException(const char* Message) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index e8da9cb90..80e1b8ee9 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -418,7 +418,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult) { - ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); + ZEN_WARN("response aborted due to error: '{}'", GetErrorAsString(IoResult)); // if one transmit failed there's really no need to go on return nullptr; -- cgit v1.2.3 From 814b2775763c60238b6c5721a843d7429750fe75 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 12:51:21 +0200 Subject: MakeWin32ErrorCode() -> MakeErrorCode() --- zencore/include/zencore/except.h | 4 ++-- zenhttp/httpsys.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 5e0f580f2..5b4a49364 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -79,9 +79,9 @@ GetLastError() } inline std::error_code -MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept +MakeErrorCode(uint32_t ErrorCode) noexcept { - return std::error_code(Win32ErrorCode, std::system_category()); + return std::error_code(ErrorCode, std::system_category()); } inline std::error_code diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 80e1b8ee9..533971c4c 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -556,7 +556,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: {}"sv, SendResult, HttpReq->pRawUrl); - ErrorCode = MakeWin32ErrorCode(SendResult); + ErrorCode = MakeErrorCode(SendResult); } else { @@ -1231,7 +1231,7 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode) // CleanupHttpIoRequest(pIoRequest); - ErrorCode = MakeWin32ErrorCode(HttpApiResult); + ErrorCode = MakeErrorCode(HttpApiResult); ZEN_ERROR("HttpReceiveHttpRequest failed, error {}", ErrorCode.message()); -- cgit v1.2.3 From 4384cfdbe11d754d6bfa011432e988dce7dd40ed Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 13:00:57 +0200 Subject: Use Memory::* API for IoBufferCore::Alloc/FreeBuffer on non-Windows --- zencore/iobuffer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 14ba77ea6..222fafa8e 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -21,12 +21,14 @@ namespace zen { void* IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) { +#if ZEN_PLATFORM_WINDOWS if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000)) { m_Flags |= kLowLevelAlloc; return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE); } else +#endif // ZEN_PLATFORM_WINDOWS { return Memory::Alloc(InSize, Alignment); } @@ -35,11 +37,13 @@ IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) void IoBufferCore::FreeBuffer() { +#if ZEN_PLATFORM_WINDOWS if (m_Flags & kLowLevelAlloc) { VirtualFree(const_cast(m_DataPtr), 0, MEM_DECOMMIT); } else +#endif // ZEN_PLATFORM_WINDOWS { return Memory::Free(const_cast(m_DataPtr)); } -- cgit v1.2.3 From f821b2d421e52e15a15e85f1bfab4b6c05b09191 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 15:44:37 +0200 Subject: Incorrect fmt format syntax causing compile errors with GCC --- zencore/iobuffer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 222fafa8e..e037abb4b 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -223,8 +223,8 @@ IoBufferExtendedCore::Materialize() const if (MappedBase == nullptr) { throw std::system_error( - std::error_code(::GetLastError(), std::system_category()), - "MapViewOfFile failed (offset {#x}, size {#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); + std::error_code(zen::GetLastError(), std::system_category()), + "MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); } m_MappedPointer = MappedBase; -- cgit v1.2.3 From 74ef06cb92d18079f5733083307e508efc85b9b3 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 16:29:45 +0200 Subject: Implemented PathFromHandle() for Linux --- zencore/filesystem.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 329cc241d..981935c55 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -605,6 +605,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr std::filesystem::path PathFromHandle(void* NativeHandle) { +#if ZEN_PLATFORM_WINDOWS if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE) { return std::filesystem::path(); @@ -618,6 +619,18 @@ PathFromHandle(void* NativeHandle) const DWORD FinalLength = GetFinalPathNameByHandleW(NativeHandle, FullPath.data(), RequiredLengthIncludingNul, FILE_NAME_OPENED); return FullPath; +#elif ZEN_PLATFORM_LINUX + char Buffer[256]; + sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); + ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + if (BytesRead <= 0) + return std::filesystem::path(); + + Buffer[BytesRead] = '\0'; + return Buffer; +#else +# error Unimplemented platform +#endif // ZEN_PLATFORM_WINDOWS } std::filesystem::path -- cgit v1.2.3 From 13761b54f48d7d4f0f6982e3726d56345ac1f739 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 16:32:08 +0200 Subject: Implemented GetRunningExecutablePath() for Linux --- zencore/filesystem.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 981935c55..4f3ee44b0 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -636,10 +636,23 @@ PathFromHandle(void* NativeHandle) std::filesystem::path GetRunningExecutablePath() { +#if ZEN_PLATFORM_WINDOWS TCHAR ExePath[MAX_PATH]; DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath)); return {std::wstring_view(ExePath, PathLength)}; +#elif ZEN_PLATFORM_LINUX + char Buffer[256]; + sprintf(Buffer, "/proc/%d/exe", getpid()); + ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + if (BytesRead < 0) + return {}; + + Buffer[BytesRead] = '\0'; + return Buffer; +#else +# error Unimplemented platform +#endif // ZEN_PLATFORM_WINDOWS } } // namespace zen -- cgit v1.2.3 From 75a1cb75d080bdc56c351c359acdef19a9865608 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 16:32:59 +0200 Subject: Added a simple filesystem test case --- zencore/filesystem.cpp | 42 ++++++++++++++++++++++++++++++++++++ zencore/include/zencore/filesystem.h | 4 ++++ zencore/zencore.cpp | 2 ++ 3 files changed, 48 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 4f3ee44b0..14eef812f 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -15,6 +15,7 @@ #include #include +#include #include namespace zen { @@ -655,4 +656,45 @@ GetRunningExecutablePath() #endif // ZEN_PLATFORM_WINDOWS } + + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +void +filesystem_forcelink() +{ +} + +TEST_CASE("filesystem") +{ + using namespace std::filesystem; + + path BinPath = GetRunningExecutablePath(); + CHECK(BinPath.stem() == "zencore-test"); + CHECK(is_regular_file(BinPath)); + + void* Handle; +#if ZEN_PLATFORM_WINDOWS + Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, 0, nullptr); + CHECK(Handle != INVALID_HANDLE_VALUE); +#else + int Fd = open(BinPath.c_str(), O_RDONLY); + CHECK(Fd >= 0); + Handle = (void*)uintptr_t(Fd); +#endif + + auto FromHandle = PathFromHandle((void*)uintptr_t(Handle)); + CHECK(equivalent(FromHandle, BinPath)); + +#if ZEN_PLATFORM_WINDOWS + CloseHandle(Handle); +#else + close(int(uintptr_t(Handle))); +#endif +} + } // namespace zen diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index a2d368d6f..a5e4dcf80 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -78,4 +78,8 @@ public: void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); }; +////////////////////////////////////////////////////////////////////////// + +void filesystem_forcelink(); // internal + } // namespace zen diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index c53fd218f..20a9e7338 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -85,6 +86,7 @@ zencore_forcelinktests() zen::blake3_forcelink(); zen::compositebuffer_forcelink(); zen::compress_forcelink(); + zen::filesystem_forcelink(); zen::intmath_forcelink(); zen::iobuffer_forcelink(); zen::memory_forcelink(); -- cgit v1.2.3 From efcf86dda10d5984815e5a7edfe42b70b2e7f5d1 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 15 Sep 2021 16:33:25 +0200 Subject: Missing include --- zencore/zencore.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 20a9e7338..56bdd2ae8 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include -- cgit v1.2.3 From 1ac503b670f293425c29d5d29a1cfbeb2e5e758d Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 09:57:55 +0200 Subject: Missing include --- zencore/iobuffer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index e037abb4b..378105e69 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include -- cgit v1.2.3 From 930b0e66b0df613c398c4cc11db03d42eef99d6b Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 11:32:35 +0200 Subject: Corrected Clang/GCC implementations of _Bsr/f --- zencore/include/zencore/intmath.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index 814a03df4..2fdea22b5 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -21,7 +21,7 @@ _BitScanReverse(unsigned long* Index, uint32_t Mask) return 0; } - *Index = __builtin_clz(Mask); + *Index = 31 - __builtin_clz(Mask); return 1; } @@ -33,8 +33,8 @@ _BitScanReverse64(unsigned long* Index, uint64_t Mask) return 0; } - *Index = __builtin_clzll(Mask); - return 0; + *Index = 63 - __builtin_clzll(Mask); + return 1; } inline uint8_t @@ -46,7 +46,7 @@ _BitScanForward64(unsigned long* Index, uint64_t Mask) } *Index = __builtin_ctzll(Mask); - return 0; + return 1; } #endif -- cgit v1.2.3 From 3f5f5f572f367052f8cb30f5545fbb0a459b25b4 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 11:36:16 +0200 Subject: Use std::fs::path::value_type instead of wchar_t for path strings --- zencore/include/zencore/iobuffer.h | 8 ++++++-- zencore/iobuffer.cpp | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index f8b8217d7..f3120983e 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -7,6 +7,8 @@ #include "refcount.h" #include "zencore.h" +#include + namespace zen { struct IoBufferExtendedCore; @@ -341,9 +343,11 @@ private: class IoBufferBuilder { + using path_char_t = std::filesystem::path::value_type; + public: - ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - ZENCORE_API static IoBuffer MakeFromTemporaryFile(const wchar_t* FileName); + ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromTemporaryFile(const path_char_t* FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } }; diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index b591dee89..4a74b9a60 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -320,7 +320,7 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) { CAtlFile DataFile; @@ -361,7 +361,7 @@ IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName) +IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName) { CAtlFile DataFile; -- cgit v1.2.3 From e0220b79488b363de6d74187f9d2f179ed4aa7ce Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:44:09 +0200 Subject: Fixed compile error if ZEN_WARN and co.'s __VA_ARGS__ was empty --- zencore/include/zencore/logging.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index 8e6b3a244..290022e2a 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -43,40 +43,40 @@ using zen::Log; do \ { \ using namespace std::literals; \ - Log().trace(fmtstr##sv, __VA_ARGS__); \ + Log().trace(fmtstr##sv, ##__VA_ARGS__);\ } while (false) #define ZEN_DEBUG(fmtstr, ...) \ do \ { \ using namespace std::literals; \ - Log().debug(fmtstr##sv, __VA_ARGS__); \ + Log().debug(fmtstr##sv, ##__VA_ARGS__);\ } while (false) #define ZEN_INFO(fmtstr, ...) \ do \ { \ using namespace std::literals; \ - Log().info(fmtstr##sv, __VA_ARGS__); \ + Log().info(fmtstr##sv, ##__VA_ARGS__);\ } while (false) #define ZEN_WARN(fmtstr, ...) \ do \ { \ using namespace std::literals; \ - Log().warn(fmtstr##sv, __VA_ARGS__); \ + Log().warn(fmtstr##sv, ##__VA_ARGS__);\ } while (false) #define ZEN_ERROR(fmtstr, ...) \ do \ { \ using namespace std::literals; \ - Log().error(fmtstr##sv, __VA_ARGS__); \ + Log().error(fmtstr##sv, ##__VA_ARGS__);\ } while (false) #define ZEN_CRITICAL(fmtstr, ...) \ do \ { \ using namespace std::literals; \ - Log().critical(fmtstr##sv, __VA_ARGS__); \ + Log().critical(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -- cgit v1.2.3 From 27c271e14e7097438999c5968e64e5cf612377dc Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 14:19:48 +0200 Subject: Whitespace adjustments to multi-line macros --- zencore/include/zencore/logging.h | 48 +++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index 290022e2a..412a39415 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -39,44 +39,44 @@ using zen::Log; // Helper macros for logging -#define ZEN_TRACE(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ +#define ZEN_TRACE(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ Log().trace(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -#define ZEN_DEBUG(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ +#define ZEN_DEBUG(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ Log().debug(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -#define ZEN_INFO(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ +#define ZEN_INFO(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ Log().info(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -#define ZEN_WARN(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().warn(fmtstr##sv, ##__VA_ARGS__);\ - } while (false) - -#define ZEN_ERROR(fmtstr, ...) \ +#define ZEN_WARN(fmtstr, ...) \ do \ { \ using namespace std::literals; \ + Log().warn(fmtstr##sv, ##__VA_ARGS__);\ + } while (false) + +#define ZEN_ERROR(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ Log().error(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -#define ZEN_CRITICAL(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ +#define ZEN_CRITICAL(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ Log().critical(fmtstr##sv, ##__VA_ARGS__);\ } while (false) -- cgit v1.2.3 From d4fa48316af92568a28dc4d56344e74eca67dd5d Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:50:25 +0200 Subject: Fixed up platform-specific includes --- zencore/filesystem.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 2942910ed..3e00f596a 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -7,12 +7,16 @@ #include #include #include -#include +#if ZEN_PLATFORM_WINDOWS +# include +#endif -#include -#include -#include -#include +#if ZEN_PLATFORM_WINDOWS +# include +# include +# include +# include +#endif #include #include -- cgit v1.2.3 From 42bac2857db915f27095588e7b0a78cd54571ebe Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:52:56 +0200 Subject: Implementation of IoBufferExtendedCore() using mmap() --- zencore/iobuffer.cpp | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 4a74b9a60..2473e0530 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -12,7 +12,12 @@ #include #include -#include +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +#endif + #include namespace zen { @@ -156,17 +161,29 @@ IoBufferExtendedCore::~IoBufferExtendedCore() { if (m_MappedPointer) { +#if ZEN_PLATFORM_WINDOWS UnmapViewOfFile(m_MappedPointer); +#else + uint64_t MapSize = ~uint64_t(uintptr_t(m_MmapHandle)); + munmap(m_MappedPointer, MapSize); +#endif } +#if ZEN_PLATFORM_WINDOWS if (m_Flags & kOwnsMmap) { CloseHandle(m_MmapHandle); } +#endif if (m_Flags & kOwnsFile) { +#if ZEN_PLATFORM_WINDOWS BOOL Success = CloseHandle(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + bool Success = (close(Fd) == 0); +#endif if (!Success) { @@ -196,6 +213,11 @@ IoBufferExtendedCore::Materialize() const if (m_MmapHandle) return; + const uint64_t MapOffset = m_FileOffset & ~0xffffull; + const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; + const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; + +#if ZEN_PLATFORM_WINDOWS m_MmapHandle = CreateFileMapping(m_FileHandle, /* lpFileMappingAttributes */ nullptr, /* flProtect */ PAGE_READONLY, @@ -211,20 +233,28 @@ IoBufferExtendedCore::Materialize() const m_Flags |= kOwnsMmap; - const uint64_t MapOffset = m_FileOffset & ~0xffffull; - const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; - const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; - void* MappedBase = MapViewOfFile(m_MmapHandle, /* dwDesiredAccess */ FILE_MAP_READ, /* FileOffsetHigh */ uint32_t(MapOffset >> 32), /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu), /* dwNumberOfBytesToMap */ MapSize); +#else + m_MmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0) + m_Flags |= kOwnsMmap; + + void* MappedBase = mmap( + /* addr */ nullptr, + /* length */ MapSize, + /* prot */ PROT_READ, + /* flags */ MAP_SHARED|MAP_NORESERVE, + /* fd */ int(uintptr_t(m_FileHandle)), + /* offset */ MapOffset); +#endif // ZEN_PLATFORM_WINDOWS if (MappedBase == nullptr) { throw std::system_error( - std::error_code(::GetLastError(), std::system_category()), + std::error_code(zen::GetLastError(), std::system_category()), "MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); } -- cgit v1.2.3 From 306de7d9a2c470613cb7697f2b0beff961fff8b5 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:54:04 +0200 Subject: MakeFromFile() on POSIX --- zencore/iobuffer.cpp | 66 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 2473e0530..16b530955 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -15,6 +15,7 @@ #if ZEN_PLATFORM_WINDOWS # include #else +# include # include #endif @@ -352,42 +353,61 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t IoBuffer IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) { + uint64_t FileSize; + +#if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - if (SUCCEEDED(hRes)) + if (FAILED(hRes)) { - ULONGLONG FileSize; - DataFile.GetSize(FileSize); + return {}; + } - // TODO: should validate that offset is in range + DataFile.GetSize((ULONGLONG&)FileSize); +#else + int Fd = open(FileName, O_RDONLY); + if (Fd < 0) + { + return {}; + } + + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + FileSize = Stat.st_size; +#endif // ZEN_PLATFORM_WINDOWS + + // TODO: should validate that offset is in range - if (Size == ~0ull) + if (Size == ~0ull) + { + Size = FileSize - Offset; + } + else + { + // Clamp size + if ((Offset + Size) > FileSize) { Size = FileSize - Offset; } - else - { - // Clamp size - if ((Offset + Size) > FileSize) - { - Size = FileSize - Offset; - } - } + } - if (Size) - { - return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size); - } - else - { - // For an empty file, we may as well just return an empty memory IoBuffer - return IoBuffer(IoBuffer::Wrap, "", 0); - } + if (Size) + { +#if ZEN_PLATFORM_WINDOWS + void* Fd = DataFile.Detach(); +#endif + return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size); } - return {}; +#if !ZEN_PLATFORM_WINDOWS + close(Fd); +#endif + + // For an empty file, we may as well just return an empty memory IoBuffer + return IoBuffer(IoBuffer::Wrap, "", 0); } IoBuffer -- cgit v1.2.3 From 572d1a61ecb2133f4854dfcfe35406699f763c05 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:56:28 +0200 Subject: MakeFromTemporaryFile() on POSIX --- zencore/iobuffer.cpp | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 16b530955..8e51f0872 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -413,6 +413,10 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint IoBuffer IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName) { + uint64_t FileSize; + void* Handle; + +#if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; // We need to open with DELETE since this is used for the case @@ -421,18 +425,33 @@ IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName) HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); - if (SUCCEEDED(hRes)) + if (FAILED(hRes)) { - ULONGLONG FileSize; - DataFile.GetSize(FileSize); + return {}; + } - IoBuffer Iob(IoBuffer::File, DataFile.Detach(), 0, FileSize); - Iob.m_Core->SetIsWholeFile(true); + DataFile.GetSize((ULONGLONG&)FileSize); - return Iob; + Handle = DataFile.Detach(); +#else + int Fd = open(FileName, O_RDONLY); + if (Fd < 0) + { + return {}; } - return {}; + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + FileSize = Stat.st_size; + + Handle = (void*)uintptr_t(Fd); +#endif // ZEN_PLATFORM_WINDOWS + + IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize); + Iob.m_Core->SetIsWholeFile(true); + + return Iob; } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From c2b761e19185f8dd608019379f894c68e7408834 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 13:56:41 +0200 Subject: Removed external linkage from g_MappingLock --- zencore/iobuffer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 8e51f0872..beb969bc7 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -195,7 +195,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore() m_DataPtr = nullptr; } -RwLock g_MappingLock; +static RwLock g_MappingLock; void IoBufferExtendedCore::Materialize() const -- cgit v1.2.3 From e4bc1374be66665d63c755c66c64f5df365fd284 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 14:02:24 +0200 Subject: Added some lawns to aid orientation within the file --- zencore/thread.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 598466bb4..620ea3bff 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -54,6 +54,8 @@ RwLock::ReleaseExclusive() #endif } +////////////////////////////////////////////////////////////////////////// + Event::Event() { m_EventHandle = CreateEvent(nullptr, true, false, nullptr); @@ -93,6 +95,8 @@ Event::Wait(int TimeoutMs) return (Result == WAIT_OBJECT_0); } +////////////////////////////////////////////////////////////////////////// + NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr) { using namespace std::literals; @@ -160,6 +164,8 @@ NamedMutex::Exists(std::string_view MutexName) return true; } +////////////////////////////////////////////////////////////////////////// + ProcessHandle::ProcessHandle() = default; void @@ -255,6 +261,8 @@ ProcessHandle::Wait(int TimeoutMs) return false; } +////////////////////////////////////////////////////////////////////////// + bool IsProcessRunning(int pid) { -- cgit v1.2.3 From df3c963bedfcda2a080c13dd63fdc4f9c6bc7691 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:33:28 +0200 Subject: The wchar_t variants of directory functions are now only enabled on Windows --- zencore/filesystem.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 3e00f596a..1ddde9d88 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -26,6 +26,8 @@ namespace zen { using namespace std::literals; +#if ZEN_PLATFORM_WINDOWS + static bool DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) { @@ -58,12 +60,6 @@ CreateDirectories(const wchar_t* Dir) return std::filesystem::create_directories(Dir); } -bool -CreateDirectories(const std::filesystem::path& Dir) -{ - return std::filesystem::create_directories(Dir); -} - // Erase all files and directories in a given directory, leaving an empty directory // behind @@ -162,6 +158,14 @@ CleanDirectory(const wchar_t* DirPath) } } +#endif // ZEN_PLATFORM_WINDOWS + +bool +CreateDirectories(const std::filesystem::path& Dir) +{ + return std::filesystem::create_directories(Dir); +} + bool DeleteDirectories(const std::filesystem::path& Dir) { -- cgit v1.2.3 From 789a34d63b8626e6b35b456ce2edd5a274f66aff Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:34:45 +0200 Subject: Non-Windows implementation of Clean/DeleteDirectory() using std::fs --- zencore/filesystem.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 1ddde9d88..fb22befdf 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -169,13 +169,35 @@ CreateDirectories(const std::filesystem::path& Dir) bool DeleteDirectories(const std::filesystem::path& Dir) { +#if ZEN_PLATFORM_WINDOWS return DeleteDirectories(Dir.c_str()); +#else + std::error_code ErrorCode; + return std::filesystem::remove_all(Dir, ErrorCode); +#endif } bool CleanDirectory(const std::filesystem::path& Dir) { +#if ZEN_PLATFORM_WINDOWS return CleanDirectory(Dir.c_str()); +#else + if (std::filesystem::exists(Dir)) + { + bool Success = true; + + std::error_code ErrorCode; + for (const auto& Item : std::filesystem::directory_iterator(Dir)) + { + Success &= std::filesystem::remove_all(Item, ErrorCode); + } + + return Success; + } + + return CreateDirectories(Dir); +#endif } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 97cfa86c2775fdfd7ddfc5d88b6283ccee5d6bbf Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:37:15 +0200 Subject: Use std::fd::path's character type for FileSysTraveral::Visitor API --- zen/cmds/run.cpp | 2 +- zencore/include/zencore/filesystem.h | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index e5210f695..97680ed5a 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -80,7 +80,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {} - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& FileName, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override { std::filesystem::path FullPath = Parent / FileName; diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index a5e4dcf80..66ab37e5c 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -69,10 +69,12 @@ class FileSystemTraversal public: struct TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0; + using path_view = std::basic_string_view; + + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0; // This should return true if we should recurse into the directory - virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0; + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0; }; void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); -- cgit v1.2.3 From fc4ec57f72d3a20df6d06cc8aa8e06ce13fabd1e Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:39:07 +0200 Subject: Stubbed out seldom-used *File() functions with a ZEN_ERROR() for Linux --- zencore/filesystem.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index fb22befdf..4b96f057e 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -205,6 +205,7 @@ CleanDirectory(const std::filesystem::path& Dir) bool SupportsBlockRefCounting(std::filesystem::path Path) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle Handle(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, @@ -231,11 +232,15 @@ SupportsBlockRefCounting(std::filesystem::path Path) } return true; +#else + return false; +#endif // ZEN_PLATFORM_WINDOWS } bool CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -390,11 +395,16 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); return AllOk; +#else + ZEN_ERROR("CloneFile() is not implemented on this platform"); + return false; +#endif // ZEN_PLATFORM_WINDOWS } bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) { +#if ZEN_PLATFORM_WINDOWS bool Success = false; if (Options.EnableClone) @@ -426,6 +436,10 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop } return Success; +#else + ZEN_ERROR("CopyFile() is not implemented on this platform"); + return false; +#endif // ZEN_PLATFORM_WINDOWS } void @@ -507,6 +521,7 @@ ReadFile(std::filesystem::path Path) bool ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function&& ProcessFunc) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -533,6 +548,10 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function Date: Thu, 16 Sep 2021 16:40:28 +0200 Subject: POSIX implementation of WriteFile() --- zencore/filesystem.cpp | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 4b96f057e..6c5c32565 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -16,7 +16,11 @@ # include # include # include +#else +# include +# include #endif + #include #include @@ -447,6 +451,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { using namespace fmt::literals; +#if ZEN_PLATFORM_WINDOWS CAtlFile Outfile; HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)) @@ -461,6 +466,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer zen::ThrowSystemException(hRes, "File open failed for '{}'"_format(Path).c_str()); } +#else + int Fd = open(Path.c_str(), O_WRONLY); + if (Fd < 0) + { + zen::CreateDirectories(Path.parent_path()); + Fd = open(Path.c_str(), O_WRONLY); + } + + if (Fd < 0) + { + ThrowLastError("File open failed for '{}'"_format(Path)); + } +#endif + // TODO: this should be block-enlightened for (size_t i = 0; i < BufferCount; ++i) @@ -472,17 +491,27 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { const uint64_t ChunkSize = zen::Min(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); +#if ZEN_PLATFORM_WINDOWS hRes = Outfile.Write(DataPtr, gsl::narrow_cast(WriteSize)); - if (FAILED(hRes)) { zen::ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str()); } +#else + if (write(Fd, DataPtr, WriteSize) != WriteSize) + { + ThrowLastError("File write failed for '{}'"_format(Path)); + } +#endif // ZEN_PLATFORM_WINDOWS WriteSize -= ChunkSize; DataPtr = reinterpret_cast(DataPtr) + ChunkSize; } } + +#if !ZEN_PLATFORM_WINDOWS + close(Fd); +#endif } void -- cgit v1.2.3 From 7e6050246217e381408b7d649d5d0edae344d8e1 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:41:12 +0200 Subject: POSIX implementation of ReadFile() --- zencore/filesystem.cpp | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 6c5c32565..a042fb7a1 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -525,6 +525,10 @@ WriteFile(std::filesystem::path Path, IoBuffer Data) FileContents ReadFile(std::filesystem::path Path) { + uint64_t FileSizeBytes; + void* Handle; + +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -538,12 +542,25 @@ ReadFile(std::filesystem::path Path) return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; } - const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart; + FileSizeBytes = FileSize.EndOfFile.QuadPart; + Handle = FromFile.Detach(); +#else + int Fd = open(Path.c_str(), O_RDONLY); + if (Fd < 0) + { + return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())}; + } - FileContents Contents; + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); - Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes)); + FileSizeBytes = Stat.st_size; + Handle = (void*)uintptr_t(Fd); +#endif + FileContents Contents; + Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes)); return Contents; } -- cgit v1.2.3 From 6ea9e1934ef3ded5be7c873dcaa89199363ea548 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:41:35 +0200 Subject: Missing include for getpid() --- zencore/filesystem.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index a042fb7a1..741e3ee19 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -19,6 +19,7 @@ #else # include # include +# include #endif #include -- cgit v1.2.3 From d39fdc301d85ea39abe331e59a49273626a98a6c Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:42:08 +0200 Subject: Implemented FileSysTraversal for Linux using open/read/closedir() --- zencore/filesystem.cpp | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 741e3ee19..a03623196 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -17,6 +17,7 @@ # include # include #else +# include # include # include # include @@ -610,6 +611,7 @@ ToUtf8(const std::filesystem::path& Path) void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { +#if ZEN_PLATFORM_WINDOWS uint64_t FileInfoBuffer[8 * 1024]; FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; @@ -697,8 +699,53 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr EntryOffset += NextOffset; } } +#else + using namespace fmt::literals; + + /* Could also implement this using Linux's getdents() syscall */ + + DIR* Dir = opendir(RootDir.c_str()); + if (Dir == nullptr) + { + ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str())); + } + + for (struct dirent* Entry; Entry = readdir(Dir);) + { + const char* FileName = Entry->d_name; + + struct stat Stat; + std::filesystem::path FullPath = RootDir / FileName; + stat(FullPath.c_str(), &Stat); + + if (S_ISDIR(Stat.st_mode)) + { + if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0) + { + /* nop */ + } + else if (Visitor.VisitDirectory(RootDir, FileName)) + { + TraverseFileSystem(FullPath, Visitor); + } + } + else if (S_ISREG(Stat.st_mode)) + { + Visitor.VisitFile(RootDir, FileName, Stat.st_size); + } + else + { + ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}", + Stat.st_mode, FileName, RootDir.c_str()); + } + } + + closedir(Dir); +#endif // ZEN_PLATFORM_WINDOWS } + + std::filesystem::path PathFromHandle(void* NativeHandle) { -- cgit v1.2.3 From 434a5ea8c5ac40530e45b2e8848ab6cb742979be Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:42:52 +0200 Subject: ToUtf8() for non-wchar_t platforms --- zencore/filesystem.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index a03623196..1cc5ff339 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -605,7 +605,11 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function Date: Thu, 16 Sep 2021 16:43:27 +0200 Subject: Simple FileSystemTraversal test case --- zencore/filesystem.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 1cc5ff339..42f0c0956 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -821,10 +821,12 @@ TEST_CASE("filesystem") { using namespace std::filesystem; + // GetExePath path BinPath = GetRunningExecutablePath(); CHECK(BinPath.stem() == "zencore-test"); CHECK(is_regular_file(BinPath)); + // PathFromHandle void* Handle; #if ZEN_PLATFORM_WINDOWS Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, @@ -844,6 +846,27 @@ TEST_CASE("filesystem") #else close(int(uintptr_t(Handle))); #endif + + // Traversal + struct : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + { + bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); + } + + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) override + { + return true; + } + + bool bFoundExpected = false; + std::filesystem::path Expected; + } Visitor; + Visitor.Expected = BinPath; + + FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); + CHECK(Visitor.bFoundExpected); } } // namespace zen -- cgit v1.2.3 From b3bab9685b82269d3a2abcd7972709f9a9e8aa55 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:44:00 +0200 Subject: There is no need to go to a wstring here as fs::path is already one --- zencore/filesystem.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index 42f0c0956..ece530474 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -621,10 +621,8 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; - std::wstring RootDirPath = RootDir.native(); - CAtlFile RootDirHandle; - HRESULT hRes = RootDirHandle.Create(RootDirPath.c_str(), + HRESULT hRes = RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, -- cgit v1.2.3 From fb7eaff9992798ce6706b1fed13569277c60d1bb Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:45:16 +0200 Subject: Make sure RootDir argument to formatted-print is a UTF8 string --- zencore/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index ece530474..c58813152 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -682,7 +682,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) { - ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir); + ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), WideToUtf8(RootDir.c_str())); } else { -- cgit v1.2.3 From c5fd2c4eff0036e7194b7260023a95f17a79d936 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:45:43 +0200 Subject: Deleted unused FilePath variable --- zencore/filesystem.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index c58813152..dca1bf6ff 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -686,8 +686,6 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else { - std::filesystem::path FullPath = RootDir / FileName; - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart); } -- cgit v1.2.3 From 30472afa4492ec07aab2ddd830f2098eaad843e1 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 16:48:27 +0200 Subject: Fixed "unreferenced formal parameter" warning --- zencore/filesystem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index dca1bf6ff..afbddcdbd 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -846,12 +846,12 @@ TEST_CASE("filesystem") // Traversal struct : public FileSystemTraversal::TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override { bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); } - virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) override + virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } -- cgit v1.2.3 From 172c737c6b92bcd39a87aa7c9576b388ffbc525c Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 17:04:11 +0200 Subject: Missing include --- zencore/include/zencore/endian.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h index d44a27b01..a0ee92d3a 100644 --- a/zencore/include/zencore/endian.h +++ b/zencore/include/zencore/endian.h @@ -2,6 +2,8 @@ #pragma once +#include + namespace zen { inline uint16_t -- cgit v1.2.3 From c3531ee1011a6027f66102c631c45fe077a09202 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 17:04:23 +0200 Subject: Added ByteSwap() tests --- zencore/intmath.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zencore/intmath.cpp b/zencore/intmath.cpp index 4039a3b39..ae65085b6 100644 --- a/zencore/intmath.cpp +++ b/zencore/intmath.cpp @@ -1,5 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include #include #include @@ -51,6 +52,10 @@ TEST_CASE("intmath") CHECK(CountTrailingZeros64(0x0000'0000'0000'0001ull) == 0); CHECK(CountTrailingZeros64(0x0000'0000'8000'0000ull) == 31); CHECK(CountTrailingZeros64(0x0000'0001'0000'0000ull) == 32); + + CHECK(ByteSwap(uint16_t(0x6d72)) == 0x726d); + CHECK(ByteSwap(uint32_t(0x2741'3965)) == 0x6539'4127); + CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); } } // namespace zen -- cgit v1.2.3 From e650a0ca8880d85fa82fcd8c776f494da05f9929 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Thu, 16 Sep 2021 17:06:27 +0200 Subject: Another missing include --- zencore/include/zencore/endian.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h index a0ee92d3a..7a9e6b44c 100644 --- a/zencore/include/zencore/endian.h +++ b/zencore/include/zencore/endian.h @@ -2,6 +2,8 @@ #pragma once +#include "zencore.h" + #include namespace zen { -- cgit v1.2.3 From c94462aa17f8e2b80a2695d5f900c4c6572085e5 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 22 Sep 2021 21:11:26 +0200 Subject: Misplaced alignas() compile fix --- zencore/include/zencore/mpscqueue.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zencore/include/zencore/mpscqueue.h b/zencore/include/zencore/mpscqueue.h index bb558bb5a..e3359852a 100644 --- a/zencore/include/zencore/mpscqueue.h +++ b/zencore/include/zencore/mpscqueue.h @@ -24,7 +24,7 @@ struct TypeCompatibleStorage ElementType* Data() { return (ElementType*)this; } const ElementType* Data() const { return (const ElementType*)this; } - char alignas(ElementType) DataMember; + alignas(ElementType) char DataMember; }; /** Fast multi-producer/single-consumer unbounded concurrent queue. -- cgit v1.2.3 From c51b4a70fc2640ef4adf682256db065a6682b88c Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 22 Sep 2021 21:11:45 +0200 Subject: Missing include compile fix --- zencore/testutils.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp index 116491950..78957cbcf 100644 --- a/zencore/testutils.cpp +++ b/zencore/testutils.cpp @@ -4,6 +4,8 @@ #include #include "zencore/string.h" +#include + namespace zen { static std::atomic Sequence{0}; -- cgit v1.2.3 From 50d5945d66ff95a7f417270240c7b9077ca556d0 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 22 Sep 2021 21:12:09 +0200 Subject: Compile out Event methods on non-Windows platforms for now --- zencore/thread.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index ded180337..30af79121 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -40,6 +40,8 @@ RwLock::ReleaseExclusive() ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS + Event::Event() { m_EventHandle = CreateEvent(nullptr, true, false, nullptr); @@ -155,6 +157,8 @@ NamedMutex::Exists(std::string_view MutexName) return true; } +#endif // ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// ProcessHandle::ProcessHandle() = default; -- cgit v1.2.3 From 65e05a02dab8595a1576511e84d0f823d90ab809 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 22 Sep 2021 21:13:33 +0200 Subject: Compile out ProcessHandle's methods on non-Windows for now --- zencore/thread.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 30af79121..5bae36b1e 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -159,6 +159,8 @@ NamedMutex::Exists(std::string_view MutexName) #endif // ZEN_PLATFORM_WINDOWS +#if ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// ProcessHandle::ProcessHandle() = default; @@ -258,6 +260,8 @@ ProcessHandle::Wait(int TimeoutMs) return false; } +#endif // ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// ProcessMonitor::ProcessMonitor() -- cgit v1.2.3 From 8df27293a17407985f0faa5b9c2cc508626037c7 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Wed, 22 Sep 2021 21:14:12 +0200 Subject: Compile out ProcessMonitor's methods on non-Windows for now --- zencore/thread.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zencore/thread.cpp b/zencore/thread.cpp index 5bae36b1e..748a0b146 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -264,6 +264,8 @@ ProcessHandle::Wait(int TimeoutMs) ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS + ProcessMonitor::ProcessMonitor() { } @@ -327,6 +329,8 @@ ProcessMonitor::IsActive() const return m_ProcessHandles.empty() == false; } +#endif // ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// bool -- cgit v1.2.3 From f6ee71a52da990b3bdc41bf56fcef87b30ca964b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:31:43 +0200 Subject: Added simple compact binary endpoint for JSON testing --- zenserver/testing/httptest.cpp | 11 +++++++++++ zenserver/testing/httptest.h | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp index c4fd6003c..18d63a6ef 100644 --- a/zenserver/testing/httptest.cpp +++ b/zenserver/testing/httptest.cpp @@ -2,6 +2,7 @@ #include "httptest.h" +#include #include namespace zen { @@ -13,6 +14,16 @@ HttpTestingService::HttpTestingService() [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, HttpVerb::kGet); + m_Router.RegisterRoute( + "json", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Obj.AddInteger("counter", ++m_Counter); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + m_Router.RegisterRoute( "echo", [this](HttpRouterRequest& Req) { diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h index b445fb450..f55780d05 100644 --- a/zenserver/testing/httptest.h +++ b/zenserver/testing/httptest.h @@ -5,6 +5,8 @@ #include #include +#include + namespace zen { /** @@ -37,7 +39,8 @@ public: }; private: - HttpRequestRouter m_Router; + HttpRequestRouter m_Router; + std::atomic m_Counter{0}; RwLock m_RwLock; std::unordered_map> m_HandlerMap; -- cgit v1.2.3 From 231a720c099ad088c53316d03cb2b6348d2cf2a7 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:32:00 +0200 Subject: clang-format --- zenstore/cidstore.cpp | 2 +- zenstore/include/zenstore/cidstore.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index d76058bd1..df5c32d25 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -46,7 +46,7 @@ struct CidStore::Impl RwLock::ExclusiveLockScope _(m_Lock); - auto It = m_CidMap.try_emplace(DecompressedId, Compressed); + auto It = m_CidMap.try_emplace(DecompressedId, Compressed); if (!It.second) { if (It.first.value() != Compressed) diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 2eea04164..5f567e7fc 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -25,9 +25,9 @@ class IoBuffer; * be used to deal with other kinds of indirections in the future. For example, if we want * to support chunking then a CID may represent a list of chunks which could be concatenated * to form the referenced chunk. - * + * * It would likely be possible to implement this mapping in a more efficient way if we - * integrate it into the CAS store itself, so we can avoid maintaining copies of large + * integrate it into the CAS store itself, so we can avoid maintaining copies of large * hashes in multiple locations. This would also allow us to consolidate commit logs etc * which would be more resilient than the current split log scheme * -- cgit v1.2.3 From dd9aec560688547dfd60090436d843c76450bd62 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:43:40 +0200 Subject: GetWindowsErrorAsString() -> GetSystemErrorAsString() --- zencore/except.cpp | 4 ++-- zencore/include/zencore/except.h | 2 +- zenhttp/httpsys.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zencore/except.cpp b/zencore/except.cpp index 44b8edffb..8614b52e1 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -72,11 +72,11 @@ ThrowSystemError(uint32_t ErrorCode, std::string_view Message) std::string GetLastErrorAsString() { - return GetWindowsErrorAsString(zen::GetLastError()); + return GetSystemErrorAsString(zen::GetLastError()); } std::string -GetWindowsErrorAsString(uint32_t Win32ErrorCode) +GetSystemErrorAsString(uint32_t Win32ErrorCode) { return std::error_code(Win32ErrorCode, std::system_category()).message(); } diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 5cfefb1e2..3430565d2 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); +ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); inline int32_t GetLastError() diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 2a50388e3..b08e39a8a 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -417,7 +417,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult != NO_ERROR) { - ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); + ZEN_WARN("response aborted due to error: '{}'", GetSystemErrorAsString(IoResult)); // if one transmit failed there's really no need to go on return nullptr; @@ -554,7 +554,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) CancelThreadpoolIo(Iocp); ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}", - GetWindowsErrorAsString(SendResult), + GetSystemErrorAsString(SendResult), HttpReq->pRawUrl, HttpReq->RequestId); -- cgit v1.2.3 From 4d06a410a2281564e90e8c4e3890d79084ed12ef Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 11:46:13 +0200 Subject: Fixed httpsys Windows compilation error --- zenhttp/httpsys.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index deaf95d5a..997491613 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -553,7 +553,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) CancelThreadpoolIo(Iocp); - ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}'", GetWindowsErrorAsString(SendResult), HttpReq->pRawUrl); + ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}'", GetErrorAsString(SendResult), HttpReq->pRawUrl); ErrorCode = MakeErrorCode(SendResult); } -- cgit v1.2.3 From cc7c8f4967d4d457652d0e43fb29dd7a358534d1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 13:03:42 +0200 Subject: vschromium: exclude vcpkg_installed directory --- vs-chromium-project.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt index 2bb89a55c..0a1075662 100644 --- a/vs-chromium-project.txt +++ b/vs-chromium-project.txt @@ -3,6 +3,7 @@ [SourceExplorer.ignore] .git/ +vcpkg_installed/ x64/ *.suo **/x64/ -- cgit v1.2.3 From 55e0ca966335fb87c136524197bfbb2346da37ac Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 13:12:37 +0200 Subject: httpsys: added `if constexpr` to silence compiler warning on Windows --- zenhttp/httpsys.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 26bba5484..747e48bb3 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -42,7 +42,7 @@ UTF8_to_wstring(const char* in) if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff)) { - if (sizeof(wchar_t) > 2) + if constexpr (sizeof(wchar_t) > 2) { out.append(1, static_cast(codepoint)); } -- cgit v1.2.3 From 43e4320ae1e760d8a30ea36f3ea22e4589d481c2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 13:12:58 +0200 Subject: httpsys: remove atl #include which is no longer needed --- zenhttp/httpsys.h | 1 - 1 file changed, 1 deletion(-) diff --git a/zenhttp/httpsys.h b/zenhttp/httpsys.h index 2e51c538f..a8395b283 100644 --- a/zenhttp/httpsys.h +++ b/zenhttp/httpsys.h @@ -17,7 +17,6 @@ # include # include "iothreadpool.h" -# include # include namespace spdlog { -- cgit v1.2.3 From c93cfbce066d481eeacdebd9eba9281ba6b66b2c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 14:48:00 +0200 Subject: CompactBinary: PLATFORM_SUPPORTS_UNALIGNED_LOADS -> ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS --- zencore/compactbinaryvalidation.cpp | 2 +- zencore/include/zencore/compactbinaryvalue.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp index 3d72148f9..a787e88ab 100644 --- a/zencore/compactbinaryvalidation.cpp +++ b/zencore/compactbinaryvalidation.cpp @@ -17,7 +17,7 @@ namespace CbValidationPrivate { template static constexpr inline T ReadUnaligned(const void* const Memory) { -#if PLATFORM_SUPPORTS_UNALIGNED_LOADS +#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS return *static_cast(Memory); #else T Value; diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h index 5795ef957..0124a8983 100644 --- a/zencore/include/zencore/compactbinaryvalue.h +++ b/zencore/include/zencore/compactbinaryvalue.h @@ -15,7 +15,7 @@ namespace CompactBinaryPrivate { template static constexpr inline T ReadUnaligned(const void* const Memory) { -#if PLATFORM_SUPPORTS_UNALIGNED_LOADS +#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS return *static_cast(Memory); #else T Value; -- cgit v1.2.3 From 250c1ce9c9df36b05dcacb8b10c6330ce8bad4cb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 16:24:13 +0200 Subject: httpsys: Added HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA to response calls which should improve overall performance (yet to be confirmed) Also added custom Server: header --- zenhttp/httpsys.cpp | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 747e48bb3..f87d3c21d 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -465,7 +465,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) HttpSendHttpResponse function as well. */ - ULONG SendFlags = 0; + ULONG SendFlags = HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA; if (m_RemainingChunkCount) { @@ -484,6 +484,23 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount); HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset; + // Server header + // + // By default this will also add a suffix " Microsoft-HTTPAPI/2.0" to this header + // + // This is controlled via a registry key 'DisableServerHeader', at: + // + // Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\HTTP\Parameters + // + // Set DisableServerHeader to 1 to disable suffix, or 2 to disable the header altogether + // + // (reference https://docs.microsoft.com/en-us/archive/blogs/dsnotes/wswcf-remove-server-header) + // + + PHTTP_KNOWN_HEADER ServerHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderServer]; + ServerHeader->pRawValue = "Zen"; + ServerHeader->RawValueLength = (USHORT)3; + // Content-length header char ContentLengthString[32]; @@ -545,11 +562,21 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ); } - if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP - && (SendResult != ERROR_IO_PENDING) // Asynchronous completion - ) + if (SendResult == NO_ERROR) + { + // Synchronous completion, but the completion event will still be posted to IOCP + + ErrorCode.clear(); + } + else if (SendResult == ERROR_IO_PENDING) + { + // Asynchronous completion, a completion notification will be posted to IOCP + + ErrorCode.clear(); + } + else { - // Some error occurred, no completion will be posted + // An error occurred, no completion will be posted to IOCP CancelThreadpoolIo(Iocp); @@ -560,10 +587,6 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ErrorCode = MakeErrorCode(SendResult); } - else - { - ErrorCode = {}; - } } /** @@ -1244,13 +1267,6 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode) { CancelThreadpoolIo(Iocp); - if (HttpApiResult == ERROR_MORE_DATA) - { - // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA); - } - - // CleanupHttpIoRequest(pIoRequest); - ErrorCode = MakeErrorCode(HttpApiResult); ZEN_ERROR("HttpReceiveHttpRequest failed, error {}", ErrorCode.message()); @@ -1258,7 +1274,7 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode) return; } - ErrorCode = std::error_code(); + ErrorCode.clear(); } HttpSysRequestHandler* -- cgit v1.2.3 From 7c1e92adea5eb9e492ec186d9913918eaf2c22e4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 17:15:16 +0200 Subject: mesh: Hide mesh functionality behind ZEN_ENABLE_MESH define This enables us to hide it from users until it's actually useful --- zenserver/config.cpp | 4 ++++ zenserver/config.h | 9 +++++---- zenserver/zenserver.cpp | 4 ++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 91fb80747..42f59b26c 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -122,12 +122,14 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value(GlobalOptions.BasePort)->default_value("1337"), ""); +#if ZEN_ENABLE_MESH options.add_option("network", "m", "mesh", "Enable mesh network", cxxopts::value(ServiceConfig.MeshEnabled)->default_value("false"), ""); +#endif options.add_option("diagnostics", "", @@ -311,7 +313,9 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv throw std::runtime_error("failed to run global config script ('{}'): {}"_format(ConfigScript, e.what()).c_str()); } +#if ZEN_ENABLE_MESH ServiceConfig.MeshEnabled = lua["mesh"]["enable"].get_or(ServiceConfig.MeshEnabled); +#endif auto UpdateStringValueFromConfig = [](const sol::table& Table, std::string_view Key, std::string& OutValue) { // Update the specified config value unless it has been set, i.e. from command line diff --git a/zenserver/config.h b/zenserver/config.h index ce059bdb2..75c19d690 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -56,10 +56,11 @@ struct ZenUpstreamCacheConfig struct ZenServiceConfig { - bool StructuredCacheEnabled = true; - bool ShouldCrash = false; // Option for testing crash handling - bool MeshEnabled = false; // Experimental p2p mesh discovery - std::string FlockId; // Id for grouping test instances into sets + bool StructuredCacheEnabled = true; + bool ShouldCrash = false; // Option for testing crash handling +#if ZEN_ENABLE_MESH + bool MeshEnabled = false; // Experimental p2p mesh discovery +#endif ZenUpstreamCacheConfig UpstreamCacheConfig; }; diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index fe4f41ab5..b45df9fef 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -261,6 +261,7 @@ public: ZEN_INFO("NOT instantiating structured cache service"); } +#if ZEN_ENABLE_MESH if (ServiceConfig.MeshEnabled) { StartMesh(BasePort); @@ -269,6 +270,7 @@ public: { ZEN_INFO("NOT starting mesh"); } +#endif m_Http = zen::CreateHttpServer(); m_Http->Initialize(BasePort); @@ -302,11 +304,13 @@ public: } } +#if ZEN_ENABLE_MESH void StartMesh(int BasePort) { ZEN_INFO("initializing mesh discovery"); m_ZenMesh.Start(uint16_t(BasePort)); } +#endif void Run() { -- cgit v1.2.3 From 3b7c392e66db63916bfb30478370214e79940158 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 17:16:02 +0200 Subject: admin: tactical checkin of some plumbing to set the stage for an actual admin interface --- zenserver/admin/admin.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++++ zenserver/admin/admin.h | 10 +++++----- zenserver/zenserver.vcxproj | 1 + 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 zenserver/admin/admin.cpp diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp new file mode 100644 index 000000000..07211cbeb --- /dev/null +++ b/zenserver/admin/admin.cpp @@ -0,0 +1,44 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "admin.h" + +#include + +namespace zen { + +HttpAdminService::HttpAdminService() +{ + m_Router.RegisterRoute( + "hello", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "health", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); +} + +HttpAdminService::~HttpAdminService() +{ +} + +const char* +HttpAdminService::BaseUri() const +{ + return "/admin/"; +} + +void +HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h index 3554b1005..6257f0998 100644 --- a/zenserver/admin/admin.h +++ b/zenserver/admin/admin.h @@ -9,14 +9,14 @@ namespace zen { class HttpAdminService : public zen::HttpService { public: - HttpAdminService() = default; - ~HttpAdminService() = default; + HttpAdminService(); + ~HttpAdminService(); - virtual const char* BaseUri() const override { return "/admin/"; } - - virtual void HandleRequest(zen::HttpServerRequest& Request) override { ZEN_UNUSED(Request); } + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; private: + HttpRequestRouter m_Router; }; } // namespace zen diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index 29436d840..bcb7ea028 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -126,6 +126,7 @@ + -- cgit v1.2.3 From 0391fe7c9917ff03730e9c90b2c0559cbe215eeb Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 17:16:30 +0200 Subject: clang-format --- zenhttp/httpsys.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index f87d3c21d..bccff24ab 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -487,11 +487,11 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) // Server header // // By default this will also add a suffix " Microsoft-HTTPAPI/2.0" to this header - // + // // This is controlled via a registry key 'DisableServerHeader', at: - // + // // Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\HTTP\Parameters - // + // // Set DisableServerHeader to 1 to disable suffix, or 2 to disable the header altogether // // (reference https://docs.microsoft.com/en-us/archive/blogs/dsnotes/wswcf-remove-server-header) -- cgit v1.2.3 From b1fd9980abf2091ece8f0f2b3ccb77b1847d4de8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 20:08:44 +0200 Subject: zencore: Added ability to forcefully set the IsInteractiveSession state --- zencore/include/zencore/zencore.h | 1 + zencore/zencore.cpp | 33 ++++++++++++++++++++++++--------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 4b9c1af1b..b5b47d076 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -198,6 +198,7 @@ ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is with ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API void SetIsInteractiveSession(bool Value); ZENCORE_API bool IsInteractiveSession(); ZENCORE_API void zencore_forcelinktests(); diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 3eb43c558..0b94f3bea 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -69,21 +69,36 @@ IsDebuggerPresent() #endif } +std::optional InteractiveSessionFlag; + +void +SetIsInteractiveSession(bool Value) +{ + InteractiveSessionFlag = Value; +} + bool IsInteractiveSession() { -#if ZEN_PLATFORM_WINDOWS - DWORD dwSessionId = 0; - if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + if (!InteractiveSessionFlag.has_value()) { - return (dwSessionId != 0); - } - - return false; +#if ZEN_PLATFORM_WINDOWS + DWORD dwSessionId = 0; + if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + { + InteractiveSessionFlag = (dwSessionId != 0); + } + else + { + InteractiveSessionFlag = false; + } #else - // TODO: figure out what makes sense here - return true; + // TODO: figure out what actually makes sense here + InteractiveSessionFlag = true; #endif + } + + return InteractiveSessionFlag.value(); } ////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From 1feafd9d186a825bbd0247d88e24e3d25934d61f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 20:09:47 +0200 Subject: zenserver: added better detection of whether we are running as a service --- zenserver/windows/service.cpp | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 017b5f9a7..b7b3b9bc1 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -3,6 +3,7 @@ #include "service.h" #include +#include #include #include @@ -146,26 +147,34 @@ CallMain(DWORD, LPSTR*) int WindowsService::ServiceMain() { - if (zen::IsInteractiveSession()) - { - // Not actually running as a service - return Run(); - } - else + gSvc = this; + + SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + + // This call returns when the service has stopped. + // The process should simply terminate when the call returns. + + if (!StartServiceCtrlDispatcher(DispatchTable)) { - gSvc = this; + const DWORD dwError = zen::GetLastError(); - SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + if (dwError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) + { + // Not actually running as a service + gSvc = nullptr; - // This call returns when the service has stopped. - // The process should simply terminate when the call returns. + zen::SetIsInteractiveSession(true); - if (!StartServiceCtrlDispatcher(DispatchTable)) + return Run(); + } + else { - SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher"); + zen::ThrowSystemError(dwError, "StartServiceCtrlDispatcher failed"); } } + zen::SetIsInteractiveSession(false); + return 0; } -- cgit v1.2.3 From 4ef4195aa9e190fd40b5b52511f44951ba897d0f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 27 Sep 2021 20:51:52 +0200 Subject: apply: Re-enabled environment variable setup for child processes --- zenserver/compute/apply.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 15d9e0141..a522aa35b 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -767,8 +767,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) lpThreadAttributes, bInheritHandles, dwCreationFlags, - nullptr, // (LPVOID)EnvironmentBlock.c_str(), // Environment block - SandboxPath.c_str(), // Current directory + (LPVOID)EnvironmentBlock.Data(), // Environment block + SandboxPath.c_str(), // Current directory &StartupInfo, /* out */ &ProcessInformation); -- cgit v1.2.3