diff options
| author | Per Larsson <[email protected]> | 2021-09-28 15:09:15 +0200 |
|---|---|---|
| committer | Per Larsson <[email protected]> | 2021-09-28 15:09:15 +0200 |
| commit | 141317786f9d59e95da8316ce40cf30e4dfd7b53 (patch) | |
| tree | 3c863384c6ca68a30e82989994408c5f40159273 | |
| parent | Removed using the bucket name to detect binary cache records and store conten... (diff) | |
| parent | apply: Re-enabled environment variable setup for child processes (diff) | |
| download | zen-141317786f9d59e95da8316ce40cf30e4dfd7b53.tar.xz zen-141317786f9d59e95da8316ce40cf30e4dfd7b53.zip | |
Merge branch 'main' of https://github.com/EpicGames/zen
33 files changed, 633 insertions, 171 deletions
diff --git a/vs-chromium-project.txt b/vs-chromium-project.txt index 2bb89a55c..0a1075662 100644 --- a/vs-chromium-project.txt +++ b/vs-chromium-project.txt @@ -3,6 +3,7 @@ [SourceExplorer.ignore] .git/ +vcpkg_installed/ x64/ *.suo **/x64/ diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index ceeb4ddae..4ffbf820c 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -82,7 +82,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {} - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& FileName, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override { std::filesystem::path FullPath = Parent / FileName; diff --git a/zencore/compactbinaryvalidation.cpp b/zencore/compactbinaryvalidation.cpp index 3d72148f9..a787e88ab 100644 --- a/zencore/compactbinaryvalidation.cpp +++ b/zencore/compactbinaryvalidation.cpp @@ -17,7 +17,7 @@ namespace CbValidationPrivate { template<typename T> static constexpr inline T ReadUnaligned(const void* const Memory) { -#if PLATFORM_SUPPORTS_UNALIGNED_LOADS +#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS return *static_cast<const T*>(Memory); #else T Value; diff --git a/zencore/except.cpp b/zencore/except.cpp index 44b8edffb..ebaecf815 100644 --- a/zencore/except.cpp +++ b/zencore/except.cpp @@ -72,13 +72,13 @@ ThrowSystemError(uint32_t ErrorCode, std::string_view Message) std::string GetLastErrorAsString() { - return GetWindowsErrorAsString(zen::GetLastError()); + return GetSystemErrorAsString(zen::GetLastError()); } std::string -GetWindowsErrorAsString(uint32_t Win32ErrorCode) +GetSystemErrorAsString(uint32_t ErrorCode) { - return std::error_code(Win32ErrorCode, std::system_category()).message(); + return std::error_code(ErrorCode, std::system_category()).message(); } #if __cpp_lib_source_location diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index f6e410ee2..d1b8b7aeb 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -7,14 +7,25 @@ #include <zencore/iobuffer.h> #include <zencore/logging.h> #include <zencore/string.h> -#include <zencore/windows.h> +#include <zencore/testing.h> -#include <atlbase.h> -#include <atlfile.h> -#include <winioctl.h> -#include <winnt.h> -#include <filesystem> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + +#if ZEN_PLATFORM_WINDOWS +# include <atlbase.h> +# include <atlfile.h> +# include <winioctl.h> +# include <winnt.h> +#else +# include <dirent.h> +# include <fcntl.h> +# include <sys/stat.h> +# include <unistd.h> +#endif +#include <filesystem> #include <gsl/gsl-lite.hpp> namespace zen { @@ -22,6 +33,7 @@ namespace zen { using namespace std::literals; #if ZEN_PLATFORM_WINDOWS + static bool DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) { @@ -48,6 +60,12 @@ DeleteReparsePoint(const wchar_t* Path, DWORD dwReparseTag) return false; } +bool +CreateDirectories(const wchar_t* Dir) +{ + return std::filesystem::create_directories(Dir); +} + // Erase all files and directories in a given directory, leaving an empty directory // behind @@ -126,12 +144,11 @@ WipeDirectory(const wchar_t* DirPath) return true; } -#endif bool -CreateDirectories(const std::filesystem::path& Dir) +DeleteDirectories(const wchar_t* DirPath) { - return std::filesystem::create_directories(Dir); + return WipeDirectory(DirPath) && RemoveDirectoryW(DirPath) == TRUE; } bool @@ -145,16 +162,46 @@ CleanDirectory(const wchar_t* DirPath) return CreateDirectories(DirPath); } +#endif // ZEN_PLATFORM_WINDOWS + +bool +CreateDirectories(const std::filesystem::path& Dir) +{ + return std::filesystem::create_directories(Dir); +} + bool DeleteDirectories(const std::filesystem::path& Dir) { - return WipeDirectory(Dir.c_str()) && RemoveDirectoryW(Dir.c_str()) == TRUE; +#if ZEN_PLATFORM_WINDOWS + return DeleteDirectories(Dir.c_str()); +#else + std::error_code ErrorCode; + return std::filesystem::remove_all(Dir, ErrorCode); +#endif } bool CleanDirectory(const std::filesystem::path& Dir) { +#if ZEN_PLATFORM_WINDOWS return CleanDirectory(Dir.c_str()); +#else + if (std::filesystem::exists(Dir)) + { + bool Success = true; + + std::error_code ErrorCode; + for (const auto& Item : std::filesystem::directory_iterator(Dir)) + { + Success &= std::filesystem::remove_all(Item, ErrorCode); + } + + return Success; + } + + return CreateDirectories(Dir); +#endif } ////////////////////////////////////////////////////////////////////////// @@ -191,12 +238,13 @@ SupportsBlockRefCounting(std::filesystem::path Path) return true; #else return false; -#endif +#endif // ZEN_PLATFORM_WINDOWS } bool CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(FromPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -351,11 +399,16 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); return AllOk; +#else + ZEN_ERROR("CloneFile() is not implemented on this platform"); + return false; +#endif // ZEN_PLATFORM_WINDOWS } bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) { +#if ZEN_PLATFORM_WINDOWS bool Success = false; if (Options.EnableClone) @@ -387,6 +440,10 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop } return Success; +#else + ZEN_ERROR("CopyFile() is not implemented on this platform"); + return false; +#endif // ZEN_PLATFORM_WINDOWS } void @@ -394,6 +451,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { using namespace fmt::literals; +#if ZEN_PLATFORM_WINDOWS CAtlFile Outfile; HRESULT hRes = Outfile.Create(Path.c_str(), GENERIC_WRITE, FILE_SHARE_READ, CREATE_ALWAYS); if (hRes == HRESULT_FROM_WIN32(ERROR_PATH_NOT_FOUND)) @@ -408,6 +466,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, "File open failed for '{}'"_format(Path).c_str()); } +#else + int Fd = open(Path.c_str(), O_WRONLY); + if (Fd < 0) + { + zen::CreateDirectories(Path.parent_path()); + Fd = open(Path.c_str(), O_WRONLY); + } + + if (Fd < 0) + { + ThrowLastError("File open failed for '{}'"_format(Path)); + } +#endif + // TODO: this should be block-enlightened for (size_t i = 0; i < BufferCount; ++i) @@ -419,17 +491,27 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer { const uint64_t ChunkSize = Min<uint64_t>(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); +#if ZEN_PLATFORM_WINDOWS hRes = Outfile.Write(DataPtr, gsl::narrow_cast<uint32_t>(WriteSize)); - if (FAILED(hRes)) { ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str()); } +#else + if (write(Fd, DataPtr, WriteSize) != WriteSize) + { + ThrowLastError("File write failed for '{}'"_format(Path)); + } +#endif // ZEN_PLATFORM_WINDOWS WriteSize -= ChunkSize; DataPtr = reinterpret_cast<const uint8_t*>(DataPtr) + ChunkSize; } } + +#if !ZEN_PLATFORM_WINDOWS + close(Fd); +#endif } void @@ -443,6 +525,10 @@ WriteFile(std::filesystem::path Path, IoBuffer Data) FileContents ReadFile(std::filesystem::path Path) { + uint64_t FileSizeBytes; + void* Handle; + +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -456,18 +542,32 @@ ReadFile(std::filesystem::path Path) return FileContents{.ErrorCode = std::error_code(::GetLastError(), std::system_category())}; } - const uint64_t FileSizeBytes = FileSize.EndOfFile.QuadPart; + FileSizeBytes = FileSize.EndOfFile.QuadPart; + Handle = FromFile.Detach(); +#else + int Fd = open(Path.c_str(), O_RDONLY); + if (Fd < 0) + { + return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())}; + } - FileContents Contents; + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); - Contents.Data.emplace_back(IoBuffer(IoBuffer::File, FromFile.Detach(), 0, FileSizeBytes)); + FileSizeBytes = Stat.st_size; + Handle = (void*)uintptr_t(Fd); +#endif + FileContents Contents; + Contents.Data.emplace_back(IoBuffer(IoBuffer::File, Handle, 0, FileSizeBytes)); return Contents; } bool ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<void(const void* Data, size_t Size)>&& ProcessFunc) { +#if ZEN_PLATFORM_WINDOWS ATL::CHandle FromFile(CreateFileW(Path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr)); if (FromFile == INVALID_HANDLE_VALUE) { @@ -494,30 +594,34 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi } return true; +#else + ZEN_ERROR("ScanFile() is not implemented on this platform"); + return false; +#endif // ZEN_PLATFORM_WINDOWS } std::string ToUtf8(const std::filesystem::path& Path) { +#if ZEN_PLATFORM_WINDOWS return WideToUtf8(Path.native().c_str()); +#else + return Path.string(); +#endif } void FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor) { +#if ZEN_PLATFORM_WINDOWS uint64_t FileInfoBuffer[8 * 1024]; FILE_INFO_BY_HANDLE_CLASS FibClass = FileIdBothDirectoryRestartInfo; bool Continue = true; - std::wstring RootDirPath = RootDir.native(); - CAtlFile RootDirHandle; - HRESULT hRes = RootDirHandle.Create(RootDirPath.c_str(), - GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS); + HRESULT hRes = + RootDirHandle.Create(RootDir.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS); if (FAILED(hRes)) { @@ -573,12 +677,12 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr } else if (DirInfo->FileAttributes & FILE_ATTRIBUTE_DEVICE) { - ZEN_WARN("encountered device node during file system traversal: {} found in {}", WideToUtf8(FileName), RootDir); + ZEN_WARN("encountered device node during file system traversal: {} found in {}", + WideToUtf8(FileName), + WideToUtf8(RootDir.c_str())); } else { - std::filesystem::path FullPath = RootDir / FileName; - Visitor.VisitFile(RootDir, FileName, DirInfo->EndOfFile.QuadPart); } @@ -592,11 +696,57 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr EntryOffset += NextOffset; } } +#else + using namespace fmt::literals; + + /* Could also implement this using Linux's getdents() syscall */ + + DIR* Dir = opendir(RootDir.c_str()); + if (Dir == nullptr) + { + ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str())); + } + + for (struct dirent* Entry; Entry = readdir(Dir);) + { + const char* FileName = Entry->d_name; + + struct stat Stat; + std::filesystem::path FullPath = RootDir / FileName; + stat(FullPath.c_str(), &Stat); + + if (S_ISDIR(Stat.st_mode)) + { + if (strcmp(FileName, ".") == 0 || strcmp(FileName, "..") == 0) + { + /* nop */ + } + else if (Visitor.VisitDirectory(RootDir, FileName)) + { + TraverseFileSystem(FullPath, Visitor); + } + } + else if (S_ISREG(Stat.st_mode)) + { + Visitor.VisitFile(RootDir, FileName, Stat.st_size); + } + else + { + ZEN_WARN("encountered non-regular file during file system traversal ({}): {} found in {}", + Stat.st_mode, + FileName, + RootDir.c_str()); + } + } + + closedir(Dir); +#endif // ZEN_PLATFORM_WINDOWS } std::filesystem::path PathFromHandle(void* NativeHandle) { +#if ZEN_PLATFORM_WINDOWS if (NativeHandle == nullptr || NativeHandle == INVALID_HANDLE_VALUE) { return std::filesystem::path(); @@ -612,15 +762,102 @@ PathFromHandle(void* NativeHandle) ZEN_UNUSED(FinalLength); return FullPath; +#elif ZEN_PLATFORM_LINUX + char Buffer[256]; + sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); + ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + if (BytesRead <= 0) + return std::filesystem::path(); + + Buffer[BytesRead] = '\0'; + return Buffer; +#else +# error Unimplemented platform +#endif // ZEN_PLATFORM_WINDOWS } std::filesystem::path GetRunningExecutablePath() { +#if ZEN_PLATFORM_WINDOWS TCHAR ExePath[MAX_PATH]; DWORD PathLength = GetModuleFileName(NULL, ExePath, ZEN_ARRAY_COUNT(ExePath)); return {std::wstring_view(ExePath, PathLength)}; +#elif ZEN_PLATFORM_LINUX + char Buffer[256]; + sprintf(Buffer, "/proc/%d/exe", getpid()); + ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + if (BytesRead < 0) + return {}; + + Buffer[BytesRead] = '\0'; + return Buffer; +#else +# error Unimplemented platform +#endif // ZEN_PLATFORM_WINDOWS +} + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +void +filesystem_forcelink() +{ } +TEST_CASE("filesystem") +{ + using namespace std::filesystem; + + // GetExePath + path BinPath = GetRunningExecutablePath(); + CHECK(BinPath.stem() == "zencore-test"); + CHECK(is_regular_file(BinPath)); + + // PathFromHandle + void* Handle; +# if ZEN_PLATFORM_WINDOWS + Handle = CreateFileW(BinPath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + CHECK(Handle != INVALID_HANDLE_VALUE); +# else + int Fd = open(BinPath.c_str(), O_RDONLY); + CHECK(Fd >= 0); + Handle = (void*)uintptr_t(Fd); +# endif + + auto FromHandle = PathFromHandle((void*)uintptr_t(Handle)); + CHECK(equivalent(FromHandle, BinPath)); + +# if ZEN_PLATFORM_WINDOWS + CloseHandle(Handle); +# else + close(int(uintptr_t(Handle))); +# endif + + // Traversal + struct : public FileSystemTraversal::TreeVisitor + { + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t) override + { + bFoundExpected |= std::filesystem::equivalent(Parent / File, Expected); + } + + virtual bool VisitDirectory(const std::filesystem::path&, const path_view&) override { return true; } + + bool bFoundExpected = false; + std::filesystem::path Expected; + } Visitor; + Visitor.Expected = BinPath; + + FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); + CHECK(Visitor.bFoundExpected); +} + +#endif + } // namespace zen diff --git a/zencore/include/zencore/compactbinaryvalue.h b/zencore/include/zencore/compactbinaryvalue.h index 5795ef957..0124a8983 100644 --- a/zencore/include/zencore/compactbinaryvalue.h +++ b/zencore/include/zencore/compactbinaryvalue.h @@ -15,7 +15,7 @@ namespace CompactBinaryPrivate { template<typename T> static constexpr inline T ReadUnaligned(const void* const Memory) { -#if PLATFORM_SUPPORTS_UNALIGNED_LOADS +#if ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS return *static_cast<const T*>(Memory); #else T Value; diff --git a/zencore/include/zencore/endian.h b/zencore/include/zencore/endian.h index d44a27b01..7a9e6b44c 100644 --- a/zencore/include/zencore/endian.h +++ b/zencore/include/zencore/endian.h @@ -2,6 +2,10 @@ #pragma once +#include "zencore.h" + +#include <cstdint> + namespace zen { inline uint16_t diff --git a/zencore/include/zencore/except.h b/zencore/include/zencore/except.h index 5cfefb1e2..e0e4aaae0 100644 --- a/zencore/include/zencore/except.h +++ b/zencore/include/zencore/except.h @@ -27,7 +27,7 @@ ZENCORE_API void ThrowLastError [[noreturn]] (std::string_view Message, const st ZENCORE_API void ThrowSystemError [[noreturn]] (uint32_t ErrorCode, std::string_view Message); ZENCORE_API std::string GetLastErrorAsString(); -ZENCORE_API std::string GetWindowsErrorAsString(uint32_t Win32ErrorCode); +ZENCORE_API std::string GetSystemErrorAsString(uint32_t Win32ErrorCode); inline int32_t GetLastError() @@ -40,9 +40,9 @@ GetLastError() } inline std::error_code -MakeWin32ErrorCode(uint32_t Win32ErrorCode) noexcept +MakeErrorCode(uint32_t ErrorCode) noexcept { - return std::error_code(Win32ErrorCode, std::system_category()); + return std::error_code(ErrorCode, std::system_category()); } inline std::error_code diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index 16d6ede53..6678528f6 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -66,13 +66,19 @@ class FileSystemTraversal public: struct TreeVisitor { - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) = 0; + using path_view = std::basic_string_view<std::filesystem::path::value_type>; + + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) = 0; // This should return true if we should recurse into the directory - virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) = 0; + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) = 0; }; void TraverseFileSystem(const std::filesystem::path& RootDir, TreeVisitor& Visitor); }; +////////////////////////////////////////////////////////////////////////// + +void filesystem_forcelink(); // internal + } // namespace zen diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index b127c8993..7619e1950 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -21,7 +21,7 @@ _BitScanReverse(unsigned long* Index, uint32_t Mask) return 0; } - *Index = __builtin_clz(Mask); + *Index = 31 - __builtin_clz(Mask); return 1; } @@ -33,8 +33,8 @@ _BitScanReverse64(unsigned long* Index, uint64_t Mask) return 0; } - *Index = __builtin_clzll(Mask); - return 0; + *Index = 63 - __builtin_clzll(Mask); + return 1; } inline uint8_t @@ -46,7 +46,7 @@ _BitScanForward64(unsigned long* Index, uint64_t Mask) } *Index = __builtin_ctzll(Mask); - return 0; + return 1; } #endif diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 21751cd40..54801f9ac 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -7,6 +7,8 @@ #include "refcount.h" #include "zencore.h" +#include <filesystem> + namespace zen { struct IoHash; @@ -371,9 +373,11 @@ private: class IoBufferBuilder { + using path_char_t = std::filesystem::path::value_type; + public: - ZENCORE_API static IoBuffer MakeFromFile(const wchar_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - ZENCORE_API static IoBuffer MakeFromTemporaryFile(const wchar_t* FileName); + ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromTemporaryFile(const path_char_t* FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) { return IoBuffer(IoBuffer::Clone, Ptr, Sz); } }; diff --git a/zencore/include/zencore/logging.h b/zencore/include/zencore/logging.h index e98509bf8..0838cfe80 100644 --- a/zencore/include/zencore/logging.h +++ b/zencore/include/zencore/logging.h @@ -39,46 +39,46 @@ using zen::Log; // Helper macros for logging -#define ZEN_TRACE(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().trace(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_TRACE(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().trace(fmtstr##sv, ##__VA_ARGS__); \ } while (false) -#define ZEN_DEBUG(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().debug(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_DEBUG(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().debug(fmtstr##sv, ##__VA_ARGS__); \ } while (false) -#define ZEN_INFO(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().info(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_INFO(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().info(fmtstr##sv, ##__VA_ARGS__); \ } while (false) -#define ZEN_WARN(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().warn(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_WARN(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().warn(fmtstr##sv, ##__VA_ARGS__); \ } while (false) -#define ZEN_ERROR(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().error(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_ERROR(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().error(fmtstr##sv, ##__VA_ARGS__); \ } while (false) -#define ZEN_CRITICAL(fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - Log().critical(fmtstr##sv, __VA_ARGS__); \ +#define ZEN_CRITICAL(fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + Log().critical(fmtstr##sv, ##__VA_ARGS__); \ } while (false) #define ZEN_CONSOLE(fmtstr, ...) \ diff --git a/zencore/include/zencore/mpscqueue.h b/zencore/include/zencore/mpscqueue.h index bb558bb5a..e3359852a 100644 --- a/zencore/include/zencore/mpscqueue.h +++ b/zencore/include/zencore/mpscqueue.h @@ -24,7 +24,7 @@ struct TypeCompatibleStorage ElementType* Data() { return (ElementType*)this; } const ElementType* Data() const { return (const ElementType*)this; } - char alignas(ElementType) DataMember; + alignas(ElementType) char DataMember; }; /** Fast multi-producer/single-consumer unbounded concurrent queue. diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index 4b9c1af1b..b5b47d076 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -198,6 +198,7 @@ ZENCORE_API bool IsPointerToStack(const void* ptr); // Query if pointer is with ZENCORE_API bool IsApplicationExitRequested(); ZENCORE_API void RequestApplicationExit(int ExitCode); ZENCORE_API bool IsDebuggerPresent(); +ZENCORE_API void SetIsInteractiveSession(bool Value); ZENCORE_API bool IsInteractiveSession(); ZENCORE_API void zencore_forcelinktests(); diff --git a/zencore/intmath.cpp b/zencore/intmath.cpp index f11f5e8aa..5a686dc8e 100644 --- a/zencore/intmath.cpp +++ b/zencore/intmath.cpp @@ -1,5 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include <zencore/endian.h> #include <zencore/intmath.h> #include <zencore/testing.h> @@ -53,6 +54,10 @@ TEST_CASE("intmath") CHECK(CountTrailingZeros64(0x0000'0000'0000'0001ull) == 0); CHECK(CountTrailingZeros64(0x0000'0000'8000'0000ull) == 31); CHECK(CountTrailingZeros64(0x0000'0001'0000'0000ull) == 32); + + CHECK(ByteSwap(uint16_t(0x6d72)) == 0x726d); + CHECK(ByteSwap(uint32_t(0x2741'3965)) == 0x6539'4127); + CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); } #endif diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index dc998d5ea..a730a316f 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -2,6 +2,7 @@ #include <zencore/iobuffer.h> +#include <zencore/except.h> #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/iohash.h> @@ -13,7 +14,13 @@ #include <memory.h> #include <system_error> -#include <atlfile.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlfile.h> +#else +# include <sys/stat.h> +# include <sys/mman.h> +#endif + #include <gsl/gsl-lite.hpp> namespace zen { @@ -23,12 +30,14 @@ namespace zen { void* IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) { +#if ZEN_PLATFORM_WINDOWS if (((InSize & 0xffFF) == 0) && (Alignment == 0x10000)) { m_Flags |= kLowLevelAlloc; return VirtualAlloc(nullptr, InSize, MEM_COMMIT, PAGE_READWRITE); } else +#endif // ZEN_PLATFORM_WINDOWS { return Memory::Alloc(InSize, Alignment); } @@ -37,11 +46,13 @@ IoBufferCore::AllocateBuffer(size_t InSize, size_t Alignment) void IoBufferCore::FreeBuffer() { +#if ZEN_PLATFORM_WINDOWS if (m_Flags & kLowLevelAlloc) { VirtualFree(const_cast<void*>(m_DataPtr), 0, MEM_DECOMMIT); } else +#endif // ZEN_PLATFORM_WINDOWS { return Memory::Free(const_cast<void*>(m_DataPtr)); } @@ -153,17 +164,29 @@ IoBufferExtendedCore::~IoBufferExtendedCore() { if (m_MappedPointer) { +#if ZEN_PLATFORM_WINDOWS UnmapViewOfFile(m_MappedPointer); +#else + uint64_t MapSize = ~uint64_t(uintptr_t(m_MmapHandle)); + munmap(m_MappedPointer, MapSize); +#endif } +#if ZEN_PLATFORM_WINDOWS if (m_Flags & kOwnsMmap) { CloseHandle(m_MmapHandle); } +#endif if (m_Flags & kOwnsFile) { +#if ZEN_PLATFORM_WINDOWS BOOL Success = CloseHandle(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + bool Success = (close(Fd) == 0); +#endif if (!Success) { @@ -174,7 +197,7 @@ IoBufferExtendedCore::~IoBufferExtendedCore() m_DataPtr = nullptr; } -RwLock g_MappingLock; +static RwLock g_MappingLock; void IoBufferExtendedCore::Materialize() const @@ -193,6 +216,11 @@ IoBufferExtendedCore::Materialize() const if (m_MmapHandle) return; + const uint64_t MapOffset = m_FileOffset & ~0xffffull; + const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; + const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; + +#if ZEN_PLATFORM_WINDOWS m_MmapHandle = CreateFileMapping(m_FileHandle, /* lpFileMappingAttributes */ nullptr, /* flProtect */ PAGE_READONLY, @@ -208,20 +236,28 @@ IoBufferExtendedCore::Materialize() const m_Flags |= kOwnsMmap; - const uint64_t MapOffset = m_FileOffset & ~0xffffull; - const uint64_t MappedOffsetDisplacement = m_FileOffset - MapOffset; - const uint64_t MapSize = m_DataBytes + MappedOffsetDisplacement; - void* MappedBase = MapViewOfFile(m_MmapHandle, /* dwDesiredAccess */ FILE_MAP_READ, /* FileOffsetHigh */ uint32_t(MapOffset >> 32), /* FileOffsetLow */ uint32_t(MapOffset & 0xffFFffFFu), /* dwNumberOfBytesToMap */ MapSize); +#else + m_MmapHandle = (void*)uintptr_t(~MapSize); // ~ so it's never null (assuming MapSize >= 0) + m_Flags |= kOwnsMmap; + + void* MappedBase = mmap( + /* addr */ nullptr, + /* length */ MapSize, + /* prot */ PROT_READ, + /* flags */ MAP_SHARED | MAP_NORESERVE, + /* fd */ int(uintptr_t(m_FileHandle)), + /* offset */ MapOffset); +#endif // ZEN_PLATFORM_WINDOWS if (MappedBase == nullptr) { throw std::system_error( - std::error_code(::GetLastError(), std::system_category()), + std::error_code(zen::GetLastError(), std::system_category()), "MapViewOfFile failed (offset {:#x}, size {:#x}) file: '{}'"_format(MapOffset, MapSize, zen::PathFromHandle(m_FileHandle))); } @@ -317,49 +353,72 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromFile(const wchar_t* FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) { + uint64_t FileSize; + +#if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); - if (SUCCEEDED(hRes)) + if (FAILED(hRes)) { - ULONGLONG FileSize; - DataFile.GetSize(FileSize); + return {}; + } + + DataFile.GetSize((ULONGLONG&)FileSize); +#else + int Fd = open(FileName, O_RDONLY); + if (Fd < 0) + { + return {}; + } + + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + FileSize = Stat.st_size; +#endif // ZEN_PLATFORM_WINDOWS - // TODO: should validate that offset is in range + // TODO: should validate that offset is in range - if (Size == ~0ull) + if (Size == ~0ull) + { + Size = FileSize - Offset; + } + else + { + // Clamp size + if ((Offset + Size) > FileSize) { Size = FileSize - Offset; } - else - { - // Clamp size - if ((Offset + Size) > FileSize) - { - Size = FileSize - Offset; - } - } + } - if (Size) - { - return IoBuffer(IoBuffer::File, DataFile.Detach(), Offset, Size); - } - else - { - // For an empty file, we may as well just return an empty memory IoBuffer - return IoBuffer(IoBuffer::Wrap, "", 0); - } + if (Size) + { +#if ZEN_PLATFORM_WINDOWS + void* Fd = DataFile.Detach(); +#endif + return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size); } - return {}; +#if !ZEN_PLATFORM_WINDOWS + close(Fd); +#endif + + // For an empty file, we may as well just return an empty memory IoBuffer + return IoBuffer(IoBuffer::Wrap, "", 0); } IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName) +IoBufferBuilder::MakeFromTemporaryFile(const path_char_t* FileName) { + uint64_t FileSize; + void* Handle; + +#if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; // We need to open with DELETE since this is used for the case @@ -368,18 +427,33 @@ IoBufferBuilder::MakeFromTemporaryFile(const wchar_t* FileName) HRESULT hRes = DataFile.Create(FileName, GENERIC_READ | DELETE, FILE_SHARE_READ | FILE_SHARE_DELETE, OPEN_EXISTING); - if (SUCCEEDED(hRes)) + if (FAILED(hRes)) { - ULONGLONG FileSize; - DataFile.GetSize(FileSize); + return {}; + } - IoBuffer Iob(IoBuffer::File, DataFile.Detach(), 0, FileSize); - Iob.m_Core->SetIsWholeFile(true); + DataFile.GetSize((ULONGLONG&)FileSize); - return Iob; + Handle = DataFile.Detach(); +#else + int Fd = open(FileName, O_RDONLY); + if (Fd < 0) + { + return {}; } - return {}; + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + FileSize = Stat.st_size; + + Handle = (void*)uintptr_t(Fd); +#endif // ZEN_PLATFORM_WINDOWS + + IoBuffer Iob(IoBuffer::File, Handle, 0, FileSize); + Iob.m_Core->SetIsWholeFile(true); + + return Iob; } IoHash diff --git a/zencore/testutils.cpp b/zencore/testutils.cpp index 116491950..78957cbcf 100644 --- a/zencore/testutils.cpp +++ b/zencore/testutils.cpp @@ -4,6 +4,8 @@ #include <zencore/session.h> #include "zencore/string.h" +#include <atomic> + namespace zen { static std::atomic<int> Sequence{0}; diff --git a/zencore/thread.cpp b/zencore/thread.cpp index d4f101454..20ab19f56 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -38,6 +38,10 @@ RwLock::ReleaseExclusive() m_Mutex.unlock(); } +////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS + Event::Event() { m_EventHandle = CreateEvent(nullptr, true, false, nullptr); @@ -84,6 +88,8 @@ Event::Wait(int TimeoutMs) return (Result == WAIT_OBJECT_0); } +////////////////////////////////////////////////////////////////////////// + NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr) { using namespace std::literals; @@ -151,6 +157,12 @@ NamedMutex::Exists(std::string_view MutexName) return true; } +#endif // ZEN_PLATFORM_WINDOWS + +#if ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////// + ProcessHandle::ProcessHandle() = default; void @@ -248,8 +260,12 @@ ProcessHandle::Wait(int TimeoutMs) return false; } +#endif // ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS + ProcessMonitor::ProcessMonitor() { } @@ -313,6 +329,8 @@ ProcessMonitor::IsActive() const return m_ProcessHandles.empty() == false; } +#endif // ZEN_PLATFORM_WINDOWS + ////////////////////////////////////////////////////////////////////////// bool diff --git a/zencore/zencore.cpp b/zencore/zencore.cpp index 1ea8ceb37..0b94f3bea 100644 --- a/zencore/zencore.cpp +++ b/zencore/zencore.cpp @@ -16,6 +16,8 @@ #include <zencore/compactbinarypackage.h> #include <zencore/compositebuffer.h> #include <zencore/compress.h> +#include <zencore/filesystem.h> +#include <zencore/intmath.h> #include <zencore/iobuffer.h> #include <zencore/memory.h> #include <zencore/mpscqueue.h> @@ -67,21 +69,36 @@ IsDebuggerPresent() #endif } +std::optional<bool> InteractiveSessionFlag; + +void +SetIsInteractiveSession(bool Value) +{ + InteractiveSessionFlag = Value; +} + bool IsInteractiveSession() { -#if ZEN_PLATFORM_WINDOWS - DWORD dwSessionId = 0; - if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + if (!InteractiveSessionFlag.has_value()) { - return (dwSessionId != 0); - } - - return false; +#if ZEN_PLATFORM_WINDOWS + DWORD dwSessionId = 0; + if (ProcessIdToSessionId(GetCurrentProcessId(), &dwSessionId)) + { + InteractiveSessionFlag = (dwSessionId != 0); + } + else + { + InteractiveSessionFlag = false; + } #else - // TODO: figure out what makes sense here - return true; + // TODO: figure out what actually makes sense here + InteractiveSessionFlag = true; #endif + } + + return InteractiveSessionFlag.value(); } ////////////////////////////////////////////////////////////////////////// @@ -109,6 +126,7 @@ zencore_forcelinktests() zen::blake3_forcelink(); zen::compositebuffer_forcelink(); zen::compress_forcelink(); + zen::filesystem_forcelink(); zen::intmath_forcelink(); zen::iobuffer_forcelink(); zen::memory_forcelink(); diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index 2a50388e3..bccff24ab 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -42,7 +42,7 @@ UTF8_to_wstring(const char* in) if (((*in & 0xc0) != 0x80) && (codepoint <= 0x10ffff)) { - if (sizeof(wchar_t) > 2) + if constexpr (sizeof(wchar_t) > 2) { out.append(1, static_cast<wchar_t>(codepoint)); } @@ -417,7 +417,7 @@ HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfB if (IoResult != NO_ERROR) { - ZEN_WARN("response aborted due to error: '{}'", GetWindowsErrorAsString(IoResult)); + ZEN_WARN("response aborted due to error: '{}'", GetSystemErrorAsString(IoResult)); // if one transmit failed there's really no need to go on return nullptr; @@ -465,7 +465,7 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) HttpSendHttpResponse function as well. */ - ULONG SendFlags = 0; + ULONG SendFlags = HTTP_SEND_RESPONSE_FLAG_BUFFER_DATA; if (m_RemainingChunkCount) { @@ -484,6 +484,23 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) HttpResponse.EntityChunkCount = USHORT(ThisRequestChunkCount); HttpResponse.pEntityChunks = m_HttpDataChunks.data() + ThisRequestChunkOffset; + // Server header + // + // By default this will also add a suffix " Microsoft-HTTPAPI/2.0" to this header + // + // This is controlled via a registry key 'DisableServerHeader', at: + // + // Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\HTTP\Parameters + // + // Set DisableServerHeader to 1 to disable suffix, or 2 to disable the header altogether + // + // (reference https://docs.microsoft.com/en-us/archive/blogs/dsnotes/wswcf-remove-server-header) + // + + PHTTP_KNOWN_HEADER ServerHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderServer]; + ServerHeader->pRawValue = "Zen"; + ServerHeader->RawValueLength = (USHORT)3; + // Content-length header char ContentLengthString[32]; @@ -545,24 +562,30 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ); } - if ((SendResult != NO_ERROR) // Synchronous completion, but the completion event will still be posted to IOCP - && (SendResult != ERROR_IO_PENDING) // Asynchronous completion - ) + if (SendResult == NO_ERROR) + { + // Synchronous completion, but the completion event will still be posted to IOCP + + ErrorCode.clear(); + } + else if (SendResult == ERROR_IO_PENDING) + { + // Asynchronous completion, a completion notification will be posted to IOCP + + ErrorCode.clear(); + } + else { - // Some error occurred, no completion will be posted + // An error occurred, no completion will be posted to IOCP CancelThreadpoolIo(Iocp); ZEN_ERROR("failed to send HTTP response (error: '{}'), request URL: '{}', request id: {}", - GetWindowsErrorAsString(SendResult), + GetSystemErrorAsString(SendResult), HttpReq->pRawUrl, HttpReq->RequestId); - ErrorCode = MakeWin32ErrorCode(SendResult); - } - else - { - ErrorCode = {}; + ErrorCode = MakeErrorCode(SendResult); } } @@ -1244,21 +1267,14 @@ InitialRequestHandler::IssueRequest(std::error_code& ErrorCode) { CancelThreadpoolIo(Iocp); - if (HttpApiResult == ERROR_MORE_DATA) - { - // ProcessReceiveAndPostResponse(pIoRequest, pServerContext->Io, ERROR_MORE_DATA); - } - - // CleanupHttpIoRequest(pIoRequest); - - ErrorCode = MakeWin32ErrorCode(HttpApiResult); + ErrorCode = MakeErrorCode(HttpApiResult); ZEN_ERROR("HttpReceiveHttpRequest failed, error {}", ErrorCode.message()); return; } - ErrorCode = std::error_code(); + ErrorCode.clear(); } HttpSysRequestHandler* diff --git a/zenhttp/httpsys.h b/zenhttp/httpsys.h index 2e51c538f..a8395b283 100644 --- a/zenhttp/httpsys.h +++ b/zenhttp/httpsys.h @@ -17,7 +17,6 @@ # include <zencore/windows.h> # include "iothreadpool.h" -# include <atlbase.h> # include <http.h> namespace spdlog { diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp new file mode 100644 index 000000000..07211cbeb --- /dev/null +++ b/zenserver/admin/admin.cpp @@ -0,0 +1,44 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "admin.h" + +#include <zencore/compactbinarybuilder.h> + +namespace zen { + +HttpAdminService::HttpAdminService() +{ + m_Router.RegisterRoute( + "hello", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "health", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); +} + +HttpAdminService::~HttpAdminService() +{ +} + +const char* +HttpAdminService::BaseUri() const +{ + return "/admin/"; +} + +void +HttpAdminService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen diff --git a/zenserver/admin/admin.h b/zenserver/admin/admin.h index 3554b1005..6257f0998 100644 --- a/zenserver/admin/admin.h +++ b/zenserver/admin/admin.h @@ -9,14 +9,14 @@ namespace zen { class HttpAdminService : public zen::HttpService { public: - HttpAdminService() = default; - ~HttpAdminService() = default; + HttpAdminService(); + ~HttpAdminService(); - virtual const char* BaseUri() const override { return "/admin/"; } - - virtual void HandleRequest(zen::HttpServerRequest& Request) override { ZEN_UNUSED(Request); } + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; private: + HttpRequestRouter m_Router; }; } // namespace zen diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 15d9e0141..a522aa35b 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -767,8 +767,8 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) lpThreadAttributes, bInheritHandles, dwCreationFlags, - nullptr, // (LPVOID)EnvironmentBlock.c_str(), // Environment block - SandboxPath.c_str(), // Current directory + (LPVOID)EnvironmentBlock.Data(), // Environment block + SandboxPath.c_str(), // Current directory &StartupInfo, /* out */ &ProcessInformation); diff --git a/zenserver/config.cpp b/zenserver/config.cpp index 91fb80747..42f59b26c 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -122,12 +122,14 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z cxxopts::value<int>(GlobalOptions.BasePort)->default_value("1337"), "<port number>"); +#if ZEN_ENABLE_MESH options.add_option("network", "m", "mesh", "Enable mesh network", cxxopts::value<bool>(ServiceConfig.MeshEnabled)->default_value("false"), ""); +#endif options.add_option("diagnostics", "", @@ -311,7 +313,9 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv throw std::runtime_error("failed to run global config script ('{}'): {}"_format(ConfigScript, e.what()).c_str()); } +#if ZEN_ENABLE_MESH ServiceConfig.MeshEnabled = lua["mesh"]["enable"].get_or(ServiceConfig.MeshEnabled); +#endif auto UpdateStringValueFromConfig = [](const sol::table& Table, std::string_view Key, std::string& OutValue) { // Update the specified config value unless it has been set, i.e. from command line diff --git a/zenserver/config.h b/zenserver/config.h index ce059bdb2..75c19d690 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -56,10 +56,11 @@ struct ZenUpstreamCacheConfig struct ZenServiceConfig { - bool StructuredCacheEnabled = true; - bool ShouldCrash = false; // Option for testing crash handling - bool MeshEnabled = false; // Experimental p2p mesh discovery - std::string FlockId; // Id for grouping test instances into sets + bool StructuredCacheEnabled = true; + bool ShouldCrash = false; // Option for testing crash handling +#if ZEN_ENABLE_MESH + bool MeshEnabled = false; // Experimental p2p mesh discovery +#endif ZenUpstreamCacheConfig UpstreamCacheConfig; }; diff --git a/zenserver/testing/httptest.cpp b/zenserver/testing/httptest.cpp index c4fd6003c..18d63a6ef 100644 --- a/zenserver/testing/httptest.cpp +++ b/zenserver/testing/httptest.cpp @@ -2,6 +2,7 @@ #include "httptest.h" +#include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> namespace zen { @@ -14,6 +15,16 @@ HttpTestingService::HttpTestingService() HttpVerb::kGet); m_Router.RegisterRoute( + "json", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Obj.AddInteger("counter", ++m_Counter); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( "echo", [this](HttpRouterRequest& Req) { IoBuffer Body = Req.ServerRequest().ReadPayload(); diff --git a/zenserver/testing/httptest.h b/zenserver/testing/httptest.h index b445fb450..f55780d05 100644 --- a/zenserver/testing/httptest.h +++ b/zenserver/testing/httptest.h @@ -5,6 +5,8 @@ #include <zencore/logging.h> #include <zenhttp/httpserver.h> +#include <atomic> + namespace zen { /** @@ -37,7 +39,8 @@ public: }; private: - HttpRequestRouter m_Router; + HttpRequestRouter m_Router; + std::atomic<uint32_t> m_Counter{0}; RwLock m_RwLock; std::unordered_map<uint32_t, Ref<PackageHandler>> m_HandlerMap; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 017b5f9a7..b7b3b9bc1 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -3,6 +3,7 @@ #include "service.h" #include <zencore/zencore.h> +#include <zencore/except.h> #include <stdio.h> #include <tchar.h> @@ -146,26 +147,34 @@ CallMain(DWORD, LPSTR*) int WindowsService::ServiceMain() { - if (zen::IsInteractiveSession()) - { - // Not actually running as a service - return Run(); - } - else + gSvc = this; + + SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + + // This call returns when the service has stopped. + // The process should simply terminate when the call returns. + + if (!StartServiceCtrlDispatcher(DispatchTable)) { - gSvc = this; + const DWORD dwError = zen::GetLastError(); - SERVICE_TABLE_ENTRY DispatchTable[] = {{(LPWSTR)SVCNAME, (LPSERVICE_MAIN_FUNCTION)&CallMain}, {NULL, NULL}}; + if (dwError == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) + { + // Not actually running as a service + gSvc = nullptr; - // This call returns when the service has stopped. - // The process should simply terminate when the call returns. + zen::SetIsInteractiveSession(true); - if (!StartServiceCtrlDispatcher(DispatchTable)) + return Run(); + } + else { - SvcReportEvent((LPTSTR)L"StartServiceCtrlDispatcher"); + zen::ThrowSystemError(dwError, "StartServiceCtrlDispatcher failed"); } } + zen::SetIsInteractiveSession(false); + return 0; } diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index fe4f41ab5..b45df9fef 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -261,6 +261,7 @@ public: ZEN_INFO("NOT instantiating structured cache service"); } +#if ZEN_ENABLE_MESH if (ServiceConfig.MeshEnabled) { StartMesh(BasePort); @@ -269,6 +270,7 @@ public: { ZEN_INFO("NOT starting mesh"); } +#endif m_Http = zen::CreateHttpServer(); m_Http->Initialize(BasePort); @@ -302,11 +304,13 @@ public: } } +#if ZEN_ENABLE_MESH void StartMesh(int BasePort) { ZEN_INFO("initializing mesh discovery"); m_ZenMesh.Start(uint16_t(BasePort)); } +#endif void Run() { diff --git a/zenserver/zenserver.vcxproj b/zenserver/zenserver.vcxproj index 29436d840..bcb7ea028 100644 --- a/zenserver/zenserver.vcxproj +++ b/zenserver/zenserver.vcxproj @@ -126,6 +126,7 @@ <ClInclude Include="windows\service.h" /> </ItemGroup> <ItemGroup> + <ClCompile Include="admin\admin.cpp" /> <ClCompile Include="cache\structuredcache.cpp" /> <ClCompile Include="cache\structuredcachestore.cpp" /> <ClCompile Include="compute\apply.cpp" /> diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index d76058bd1..df5c32d25 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -46,7 +46,7 @@ struct CidStore::Impl RwLock::ExclusiveLockScope _(m_Lock); - auto It = m_CidMap.try_emplace(DecompressedId, Compressed); + auto It = m_CidMap.try_emplace(DecompressedId, Compressed); if (!It.second) { if (It.first.value() != Compressed) diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 2eea04164..5f567e7fc 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -25,9 +25,9 @@ class IoBuffer; * be used to deal with other kinds of indirections in the future. For example, if we want * to support chunking then a CID may represent a list of chunks which could be concatenated * to form the referenced chunk. - * + * * It would likely be possible to implement this mapping in a more efficient way if we - * integrate it into the CAS store itself, so we can avoid maintaining copies of large + * integrate it into the CAS store itself, so we can avoid maintaining copies of large * hashes in multiple locations. This would also allow us to consolidate commit logs etc * which would be more resilient than the current split log scheme * |