diff options
90 files changed, 1623 insertions, 441 deletions
diff --git a/.gitignore b/.gitignore index bf237a7b3..103d3bd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ +vsxmake20*/ +vs20*/ # Visual Studio Code options directory .vscode/ @@ -217,6 +219,7 @@ __pycache__/ .minio_data/ .test/ .xmake/ +.gdb_history # Tags TAGS @@ -55,11 +55,67 @@ You can then build from the command line: It's also possible to generate project files for Visual Studio via -`d:\zen> xmake project -k vsxmake` +`d:\zen> xmake project -k vsxmake -a x64` ## Building on Linux -... coming soon +The following instructions have been collated using Ubuntu 20.04. + +At the time of writing only GCC v11 supports the C++20 features used by Zen. As +this version is not available in the default package repositories we need to +get GCC from one of Ubuntu's toolchain repositories; + +``` +sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test +sudo apt install -y gcc-11 +gcc-11 --version +``` + +Next we need the `xmake` build system. For this we will download and install +`xmake` as a `.deb` package. Check the project's release page for more up to +date `.deb` files; + +``` +wget https://github.com/xmake-io/xmake/releases/download/v2.5.7/xmake-v2.5.7.amd64.deb +sudo dpkg -i xmake-v2.5.7.amd64.deb +xmake --version +``` + +For some of Zen's third party dependencies are provided by Microsoft's `vcpkg` +C++ library manager. After cloning the project there is a initialisation step; + +``` +git clone https://github.com/microsoft/vcpkg.git ~/zen/vcpkg +~/zen/vcpkg/bootstrap-vcpkg.sh +``` + +`xmake` uses an environment variable to find `vcpkg`. Alternatively this can be +done by including `VCPKG_ROOT=...` on the command line when invoking `xmake`; + +``` +export VCPKG_ROOT=~/zen/vcpkg +``` + +Clone the Zen project and tell `xmake` to use the correct GCC version; + +``` +git clone https://github.com/EpicGames/zen.git ~/zen/main +cd ~/zen/main +xmake config --plat=linux --cxx=g++-11 --cc=gcc-11 [--mode=debug] +``` + +The `--mode=debug` is optionally required to change the build variant. This +cannot be done when executing the build and the setting of configuration options +can not be composed across multiple `xmake config` calls. + +Now we are ready to build Zen. The `-y` skips `xmake` from prompting about +updating `vcpkg` packages; + +``` +xmake build -y +``` + +The `xmake` flags `-vD` can be useful to diagnose `xmake` issues. ## Building on Mac @@ -1,23 +1,25 @@ add_requires( - "vcpkg::doctest", - "vcpkg::spdlog", - "vcpkg::gsl-lite", - "vcpkg::asio", - "vcpkg::cpr", - "vcpkg::xxhash", - "vcpkg::robin-map", - "vcpkg::lz4", - "vcpkg::fmt", - "vcpkg::cxxopts", - "vcpkg::mimalloc", - "vcpkg::sol2", - "vcpkg::sentry-native", + "vcpkg::asio", + "vcpkg::cpr", + "vcpkg::curl", + "vcpkg::cxxopts", + "vcpkg::doctest", + "vcpkg::fmt", + "vcpkg::gsl-lite", + "vcpkg::http-parser", "vcpkg::json11", - "vcpkg::lua", - "vcpkg::curl", - "vcpkg::zlib", - "vcpkg::zstd", - "vcpkg::http-parser") + "vcpkg::lua", + "vcpkg::lz4", + "vcpkg::mimalloc", + "vcpkg::openssl", + "vcpkg::robin-map", + "vcpkg::sentry-native", + "vcpkg::sol2", + "vcpkg::spdlog", + "vcpkg::xxhash", + "vcpkg::zlib", + "vcpkg::zstd" +) add_rules("mode.debug", "mode.release") @@ -39,27 +41,50 @@ else end if is_os("windows") then - add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00") + add_defines( + "_CRT_SECURE_NO_WARNINGS", + "_UNICODE", + "UNICODE", + "_CONSOLE", + "NOMINMAX", -- stop Windows SDK defining 'min' and 'max' + "NOGDI", -- otherwise Windows.h defines 'GetObject' + "WIN32_LEAN_AND_MEAN", -- cut down Windows.h + "_WIN32_WINNT=0x0A00" + ) -- add_ldflags("/MAP") end +if is_os("linux") then + add_cxxflags("-Wno-unused-value") + add_cxxflags("-Wno-strict-aliasing") + add_cxxflags("-Wno-implicit-fallthrough") + add_cxxflags("-Wno-missing-field-initializers") +end + add_defines("USE_SENTRY=1") add_defines("ZEN_USE_MIMALLOC=1") -option("vfs") - set_showmenu(true) - set_description("Enable VFS functionality") - add_defines("ZEN_WITH_VFS") -option_end() +if is_os("windows") then + option("vfs") + set_showmenu(true) + set_description("Enable VFS functionality") + add_defines("ZEN_WITH_VFS") + option_end() -option("httpsys") - set_default(true) - set_showmenu(true) - set_description("Enable http.sys server") - add_defines("ZEN_WITH_HTTPSYS") -option_end() + option("httpsys") + set_default(true) + set_showmenu(true) + set_description("Enable http.sys server") + add_defines("ZEN_WITH_HTTPSYS") + option_end() -add_defines("UNICODE", "_CONSOLE") + option("compute") + set_default(true) + set_showmenu(true) + set_description("Enable compute services endpoint") + add_defines("ZEN_WITH_COMPUTE_SERVICES") + option_end() +end set_warnings("allextra", "error") set_languages("cxx20") diff --git a/zen/cmds/dedup.cpp b/zen/cmds/dedup.cpp index 089212ed9..2f6e5d2cc 100644 --- a/zen/cmds/dedup.cpp +++ b/zen/cmds/dedup.cpp @@ -10,7 +10,9 @@ #include <zencore/thread.h> #include <zencore/timer.h> -#include <ppl.h> +#if ZEN_PLATFORM_WINDOWS +# include <ppl.h> +#endif namespace zen { diff --git a/zen/cmds/dedup.h b/zen/cmds/dedup.h index 7932d10e6..dbda236f6 100644 --- a/zen/cmds/dedup.h +++ b/zen/cmds/dedup.h @@ -4,8 +4,6 @@ #include "../zen.h" -#include <ppl.h> - namespace zen { /** Deduplicate files in a tree using block cloning diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 8cbf13566..085e74061 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -152,7 +152,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (auto It = Visit.m_HashToFile.find(NeedHash); It != Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url("http://localhost:13337/cas"), cpr::Body((const char*)FileData.Data(), FileData.Size())); diff --git a/zen/internalfile.cpp b/zen/internalfile.cpp index 3ad5a7d05..32108f615 100644 --- a/zen/internalfile.cpp +++ b/zen/internalfile.cpp @@ -5,7 +5,6 @@ #include <zencore/filesystem.h> #include <zencore/fmtutils.h> #include <zencore/logging.h> -#include <zencore/windows.h> #include <gsl/gsl-lite.hpp> diff --git a/zen/internalfile.h b/zen/internalfile.h index c6071418e..08083678f 100644 --- a/zen/internalfile.h +++ b/zen/internalfile.h @@ -10,9 +10,14 @@ #include <zencore/iobuffer.h> #include <zencore/refcount.h> #include <zencore/thread.h> -#include <zencore/windows.h> -#include <atlfile.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + +#if ZEN_PLATFORM_WINDOWS +# include <atlfile.h> +#endif #include <filesystem> #include <list> diff --git a/zen/zen.cpp b/zen/zen.cpp index 3c33ff5e0..1fe23d32b 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -264,7 +264,7 @@ main(int argc, char** argv) for (const CommandInfo& CmdInfo : Commands) { - if (_stricmp(SubCommand.c_str(), CmdInfo.CmdName) == 0) + if (StrCaseCompare(SubCommand.c_str(), CmdInfo.CmdName) == 0) { cxxopts::Options* VerbOptions = CmdInfo.Cmd->Options(); @@ -2,13 +2,16 @@ #pragma once -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +#include <zencore/refcount.h> +#include <zencore/zencore.h> + +ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END -#include <zencore/refcount.h> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif #include <filesystem> diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp index 8f8952271..b2019d4e2 100644 --- a/zencore/blake3.cpp +++ b/zencore/blake3.cpp @@ -8,7 +8,9 @@ #include <zencore/zencore.h> #include "../thirdparty/BLAKE3/c/blake3.h" -#pragma comment(lib, "blake3.lib") +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "blake3.lib") +#endif #include <string.h> diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index c6bf38b04..86ee6e022 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -18,6 +18,8 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> +#else +# include <time.h> #endif #if ZEN_WITH_TESTS @@ -46,15 +48,31 @@ IsLeapYear(int Year) return false; } +static constexpr uint64_t +GetPlatformToDateTimeBiasInSeconds() +{ +#if ZEN_PLATFORM_WINDOWS + const uint64_t PlatformEpochYear = 1601; +#else + const uint64_t PlatformEpochYear = 1970; +#endif + const uint64_t DateTimeEpochYear = 1; + return uint64_t(double(PlatformEpochYear - DateTimeEpochYear) * 365.2425) * 86400; +} + DateTime DateTime::Now() { + static const uint64_t EpochBias = GetPlatformToDateTimeBiasInSeconds(); + static const uint64_t SecsTo100nsTicks = int64_t(10e9 / 100); + #if ZEN_PLATFORM_WINDOWS FILETIME SysTime; GetSystemTimeAsFileTime(&SysTime); - return DateTime{504911232000000000ull + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime}; + return DateTime{(EpochBias * SecsTo100nsTicks) + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime}; #else -# error Needs implementation + int64_t SecondsSinceUnixEpoch = time(nullptr); + return DateTime{(EpochBias + SecondsSinceUnixEpoch) * SecsTo100nsTicks}; #endif } @@ -82,88 +100,6 @@ DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, in Second * TimeSpan::TicksPerSecond + MilliSecond * TimeSpan::TicksPerMillisecond; } -void -TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) -{ - int64_t TotalTicks = 0; - - TotalTicks += Days * TicksPerDay; - TotalTicks += Hours * TicksPerHour; - TotalTicks += Minutes * TicksPerMinute; - TotalTicks += Seconds * TicksPerSecond; - TotalTicks += FractionNano / NanosecondsPerTick; - - Ticks = TotalTicks; -} - -std::string -TimeSpan::ToString(const char* Format) const -{ - using namespace fmt::literals; - - StringBuilder<128> Result; - - Result.Append((Ticks < 0) ? '-' : '+'); - - while (*Format != '\0') - { - if ((*Format == '%') && (*++Format != '\0')) - { - switch (*Format) - { - case 'd': - Result.Append("{}"_format(GetDays())); - break; - case 'D': - Result.Append("{:08}"_format(GetDays())); - break; - case 'h': - Result.Append("{:02}"_format(GetHours())); - break; - case 'm': - Result.Append("{:02}"_format(GetMinutes())); - break; - case 's': - Result.Append("{:02}"_format(GetSeconds())); - break; - case 'f': - Result.Append("{:03}"_format(GetFractionMilli())); - break; - case 'u': - Result.Append("{:06}"_format(GetFractionMicro())); - break; - case 't': - Result.Append("{:07}"_format(GetFractionTicks())); - break; - case 'n': - Result.Append("{:09}"_format(GetFractionNano())); - break; - default: - Result.Append(*Format); - } - } - else - { - Result.Append(*Format); - } - - ++Format; - } - - return Result.ToString(); -} - -std::string -TimeSpan::ToString() const -{ - if (GetDays() == 0) - { - return ToString("%h:%m:%s.%f"); - } - - return ToString("%d.%h:%m:%s.%f"); -} - int DateTime::GetYear() const { @@ -329,6 +265,88 @@ DateTime::ToIso8601() const return ToString("%Y-%m-%dT%H:%M:%S.%sZ"); } +void +TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) +{ + int64_t TotalTicks = 0; + + TotalTicks += Days * TicksPerDay; + TotalTicks += Hours * TicksPerHour; + TotalTicks += Minutes * TicksPerMinute; + TotalTicks += Seconds * TicksPerSecond; + TotalTicks += FractionNano / NanosecondsPerTick; + + Ticks = TotalTicks; +} + +std::string +TimeSpan::ToString(const char* Format) const +{ + using namespace fmt::literals; + + StringBuilder<128> Result; + + Result.Append((int64_t(Ticks) < 0) ? '-' : '+'); + + while (*Format != '\0') + { + if ((*Format == '%') && (*++Format != '\0')) + { + switch (*Format) + { + case 'd': + Result.Append("{}"_format(GetDays())); + break; + case 'D': + Result.Append("{:08}"_format(GetDays())); + break; + case 'h': + Result.Append("{:02}"_format(GetHours())); + break; + case 'm': + Result.Append("{:02}"_format(GetMinutes())); + break; + case 's': + Result.Append("{:02}"_format(GetSeconds())); + break; + case 'f': + Result.Append("{:03}"_format(GetFractionMilli())); + break; + case 'u': + Result.Append("{:06}"_format(GetFractionMicro())); + break; + case 't': + Result.Append("{:07}"_format(GetFractionTicks())); + break; + case 'n': + Result.Append("{:09}"_format(GetFractionNano())); + break; + default: + Result.Append(*Format); + } + } + else + { + Result.Append(*Format); + } + + ++Format; + } + + return Result.ToString(); +} + +std::string +TimeSpan::ToString() const +{ + if (GetDays() == 0) + { + return ToString("%h:%m:%s.%f"); + } + + return ToString("%d.%h:%m:%s.%f"); +} + StringBuilderBase& Guid::ToString(StringBuilderBase& Sb) const { diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp index fa5b6a69b..3be24c149 100644 --- a/zencore/compactbinarybuilder.cpp +++ b/zencore/compactbinarybuilder.cpp @@ -985,7 +985,12 @@ TEST_CASE("usonbuilder.string") SUBCASE("Non-ASCII String") { +#if ZEN_SIZEOF_WCHAR_T == 2 wchar_t Value[2] = {0xd83d, 0xde00}; +#else + wchar_t Value[1] = {0x1f600}; +#endif + Writer.AddString("\xf0\x9f\x98\x80"sv); Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value))); CbFieldIterator Fields = Writer.Save(); diff --git a/zencore/compress.cpp b/zencore/compress.cpp index 4b6d83f6f..a1a6731ff 100644 --- a/zencore/compress.cpp +++ b/zencore/compress.cpp @@ -1198,7 +1198,6 @@ TEST_CASE("CompressedBuffer") SUBCASE("copy uncompressed range") { - const uint64_t BlockSize = 64 * sizeof(uint64_t); const uint64_t N = 1000; std::vector<uint64_t> ExpectedValues = GenerateData(N); diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index a642e2cf6..3c5ab2026 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -174,7 +174,8 @@ CleanDirectory(const wchar_t* DirPath) bool CreateDirectories(const std::filesystem::path& Dir) { - return std::filesystem::create_directories(Dir); + std::error_code ErrorCode; + return std::filesystem::create_directories(Dir, ErrorCode); } bool @@ -244,6 +245,7 @@ SupportsBlockRefCounting(std::filesystem::path Path) return true; #else + ZEN_UNUSED(Path); return false; #endif // ZEN_PLATFORM_WINDOWS } @@ -406,16 +408,47 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); return AllOk; -#else +#elif ZEN_PLATFORM_LINUX +# if 0 + struct ScopedFd + { + ~ScopedFd() { close(Fd); } + int Fd; + }; + + // The 'from' file + int FromFd = open(FromPath.c_str(), O_RDONLY); + if (FromFd < 0) + { + return false; + } + ScopedFd $From = { FromFd }; + + // The 'to' file + int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL, 0666); + if (ToFd < 0) + { + return false; + } + ScopedFd $To = { FromFd }; + + ioctl(ToFd, FICLONE, FromFd); + + return false; +# endif // 0 + ZEN_UNUSED(FromPath, ToPath); ZEN_ERROR("CloneFile() is not implemented on this platform"); return false; +#elif ZEN_PLATFORM_MAC + /* clonefile() syscall if APFS */ +# error not implemented + return false; #endif // ZEN_PLATFORM_WINDOWS } bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) { -#if ZEN_PLATFORM_WINDOWS bool Success = false; if (Options.EnableClone) @@ -433,6 +466,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop return false; } +#if ZEN_PLATFORM_WINDOWS BOOL CancelFlag = FALSE; Success = !!::CopyFileExW(FromPath.c_str(), ToPath.c_str(), @@ -440,17 +474,54 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop /* lpData */ nullptr, &CancelFlag, /* dwCopyFlags */ 0); +#else + using namespace fmt::literals; - if (!Success) + struct ScopedFd { ~ScopedFd() { close(Fd); } int Fd; }; + + // From file + int FromFd = open(FromPath.c_str(), O_RDONLY); + if (FromFd < 0) { - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed"); + ThrowLastError("failed to open file {}"_format(FromPath)); } + ScopedFd $From = { FromFd }; - return Success; -#else - ZEN_ERROR("CopyFile() is not implemented on this platform"); - return false; + // To file + int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL); + if (ToFd < 0) + { + ThrowLastError("failed to create file {}"_format(ToPath)); + } + ScopedFd $To = { ToFd }; + + // Copy impl + static const size_t BufferSize = 64 << 10; + void* Buffer = malloc(BufferSize); + while (true) + { + int BytesRead = read(FromFd, Buffer, BufferSize); + if (BytesRead <= 0) + { + Success = (BytesRead == 0); + break; + } + + if (write(ToFd, Buffer, BytesRead) != BufferSize) + { + Success = false; + break; + } + } + free(Buffer); #endif // ZEN_PLATFORM_WINDOWS + + if (!Success) + { + ThrowLastError("file copy failed"sv); + } + + return true; } void @@ -505,7 +576,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != WriteSize) + if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) { ThrowLastError("File write failed for '{}'"_format(Path)); } @@ -572,7 +643,9 @@ ReadFile(std::filesystem::path Path) int Fd = open(Path.c_str(), O_RDONLY); if (Fd < 0) { - return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())}; + FileContents Ret; + Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category()); + return Ret; } static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); @@ -616,12 +689,43 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi ProcessFunc(ReadBuffer.data(), dwBytesRead); } - - return true; #else - ZEN_ERROR("ScanFile() is not implemented on this platform"); - return false; + int Fd = open(Path.c_str(), O_RDONLY); + if (Fd < 0) + { + return false; + } + + bool Success = true; + + void* Buffer = malloc(ChunkSize); + while (true) + { + int BytesRead = read(Fd, Buffer, ChunkSize); + if (BytesRead < 0) + { + Success = false; + break; + } + + if (BytesRead == 0) + { + break; + } + + ProcessFunc(Buffer, BytesRead); + } + + free(Buffer); + close(Fd); + + if (!Success) + { + ThrowLastError("file scan failed"); + } #endif // ZEN_PLATFORM_WINDOWS + + return true; } std::string @@ -731,7 +835,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str())); } - for (struct dirent* Entry; Entry = readdir(Dir);) + for (struct dirent* Entry; (Entry = readdir(Dir));) { const char* FileName = Entry->d_name; @@ -787,14 +891,15 @@ PathFromHandle(void* NativeHandle) return FullPath; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead <= 0) return std::filesystem::path(); - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS @@ -809,14 +914,15 @@ GetRunningExecutablePath() return {std::wstring_view(ExePath, PathLength)}; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/exe", getpid()); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/exe", getpid()); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead < 0) return {}; - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS @@ -881,6 +987,50 @@ TEST_CASE("filesystem") FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); CHECK(Visitor.bFoundExpected); + + // Scan/read file + FileContents BinRead = ReadFile(BinPath); + std::vector<uint8_t> BinScan; + ScanFile(BinPath, 16 << 10, [&] (const void* Data, size_t Size) { + const auto* Ptr = (uint8_t*)Data; + BinScan.insert(BinScan.end(), Ptr, Ptr + Size); + }); + CHECK_EQ(BinRead.Data.size(), 1); + CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); +} + +TEST_CASE("PathBuilder") +{ +#if ZEN_PLATFORM_WINDOWS + const char* foo_bar = "/foo\\bar"; +#else + const char* foo_bar = "/foo/bar"; +#endif + + ExtendablePathBuilder<32> Path; + for (const char* Prefix : { "/foo", "/foo/" }) + { + Path.Reset(); + Path.Append(Prefix); + Path /= "bar"; + CHECK(Path.ToPath() == foo_bar); + } + + using fspath = std::filesystem::path; + + Path.Reset(); + Path.Append(fspath("/foo/")); + Path /= (fspath("bar")); + CHECK(Path.ToPath() == foo_bar); + +#if ZEN_PLATFORM_WINDOWS + Path.Reset(); + Path.Append(fspath(L"/\u0119oo/")); + Path /= L"bar"; + printf("%ls\n", Path.ToPath().c_str()); + CHECK(Path.ToView() == L"/\u0119oo/bar"); + CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); +# endif } #endif diff --git a/zencore/include/zencore/blockingqueue.h b/zencore/include/zencore/blockingqueue.h index 277095689..f92df5a54 100644 --- a/zencore/include/zencore/blockingqueue.h +++ b/zencore/include/zencore/blockingqueue.h @@ -3,6 +3,7 @@ #pragma once #include <atomic> +#include <condition_variable> #include <deque> #include <mutex> diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index 06331c510..0be23ea11 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -63,7 +63,7 @@ public: private: void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); - uint64_t Ticks; + uint64_t Ticks; // 1 tick == 0.1us == 100ns, epoch == Jan 1st 0001 }; class TimeSpan diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index c7ac7140d..ec857b33c 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -5,6 +5,7 @@ #include "zencore.h" #include <zencore/iobuffer.h> +#include <zencore/string.h> #include <filesystem> #include <functional> @@ -57,6 +58,70 @@ ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path); +extern template class StringBuilderImpl<std::filesystem::path::value_type>; + +/** + * Helper class for building paths. Backed by a string builder. + * + */ +class PathBuilderBase + : public StringBuilderImpl<std::filesystem::path::value_type> +{ +private: + using Super = StringBuilderImpl<std::filesystem::path::value_type>; + +protected: + using CharType = std::filesystem::path::value_type; + using ViewType = std::basic_string_view<CharType>; + +public: + void Append(const std::filesystem::path& Rhs) { Super::Append(Rhs.c_str()); } + void operator /= (const std::filesystem::path& Rhs) { this->operator /= (Rhs.c_str()); }; + void operator /= (const CharType* Rhs) { AppendSeparator(); Super::Append(Rhs); } + operator ViewType () const { return ToView(); } + std::basic_string_view<CharType> ToView() const { return std::basic_string_view<CharType>(Data(), Size()); } + std::filesystem::path ToPath() const { return std::filesystem::path(ToView()); } + + std::string ToUtf8() const + { +#if ZEN_PLATFORM_WINDOWS + return WideToUtf8(ToView()); +#else + return std::string(ToView()); +#endif + } + + void AppendSeparator() + { + if ( + ToView().ends_with(std::filesystem::path::preferred_separator) +#if ZEN_PLATFORM_WINDOWS + || ToView().ends_with('/') +#endif + ) + return; + + Super::Append(std::filesystem::path::preferred_separator); + } +}; + +template <size_t N> +class PathBuilder : public PathBuilderBase +{ +public: + PathBuilder() { Init(m_Buffer, N); } + +private: + PathBuilderBase::CharType m_Buffer[N]; +}; + +template <size_t N> +class ExtendablePathBuilder : public PathBuilder<N> +{ +public: + ExtendablePathBuilder() { this->m_IsExtendable = true; } +}; + /** * Efficient file system traversal * diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index 7619e1950..0d0ceff16 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -107,7 +107,7 @@ FloorLog2(uint32_t Value) static inline uint32_t CountLeadingZeros(uint32_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); return 32 - Log2; } @@ -115,7 +115,7 @@ CountLeadingZeros(uint32_t Value) static inline uint64_t FloorLog2_64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return Log2 & Mask; } @@ -123,7 +123,7 @@ FloorLog2_64(uint64_t Value) static inline uint64_t CountLeadingZeros64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return ((63 - Log2) & Mask) | (64 & ~Mask); } diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 04b3b33dd..fee89a408 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -374,10 +374,8 @@ private: class IoBufferBuilder { - using path_char_t = std::filesystem::path::value_type; - public: - ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h index 7167ab3b5..92ebebcfe 100644 --- a/zencore/include/zencore/refcount.h +++ b/zencore/include/zencore/refcount.h @@ -159,7 +159,7 @@ public: private: T* m_Ref = nullptr; - template<class T> + template<class U> friend class Ref; }; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index a5ab25068..7af27ca20 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -424,7 +424,7 @@ public: inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } inline std::wstring toString() const { return std::wstring{Data(), Size()}; } - inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); } inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } using StringBuilderImpl:: operator<<; }; @@ -460,7 +460,6 @@ std::wstring Utf8ToWide(const std::string_view& wstr); void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); std::string WideToUtf8(const wchar_t* wstr); -void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out); void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); std::string WideToUtf8(const std::wstring_view Wstr); @@ -697,6 +696,19 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) ////////////////////////////////////////////////////////////////////////// +inline int32_t +StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length=-1) +{ + // A helper for cross-platform case-insensitive string comparison. +#if ZEN_PLATFORM_WINDOWS + return (Length < 0) ? _stricmp(Lhs, Rhs) : _strnicmp(Lhs, Rhs, size_t(Length)); +#else + return (Length < 0) ? strcasecmp(Lhs, Rhs) : strncasecmp(Lhs, Rhs, size_t(Length)); +#endif +} + +////////////////////////////////////////////////////////////////////////// + /** * ASCII character bitset useful for fast and readable parsing * diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index 9fc4c87a2..6727e5029 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -102,11 +102,35 @@ protected: /** Basic abstraction of an IPC mechanism (aka 'binary semaphore') */ -class NamedEvent : public Event +class NamedEvent { public: - ZENCORE_API explicit NamedEvent(std::string_view EventName); - ZENCORE_API explicit NamedEvent(std::u8string_view EventName); + NamedEvent() = default; + ZENCORE_API explicit NamedEvent(std::string_view EventName); + ZENCORE_API ~NamedEvent(); + ZENCORE_API void Close(); + ZENCORE_API void Set(); + ZENCORE_API bool Wait(int TimeoutMs=-1); + + NamedEvent(NamedEvent&& Rhs) noexcept + : m_EventHandle(Rhs.m_EventHandle) + { + Rhs.m_EventHandle = nullptr; + } + + inline NamedEvent& operator = (NamedEvent&& Rhs) noexcept + { + std::swap(m_EventHandle, Rhs.m_EventHandle); + return *this; + } + + +protected: + void* m_EventHandle = nullptr; + +private: + NamedEvent(const NamedEvent& Rhs) = delete; + NamedEvent& operator = (const NamedEvent& Rhs) = delete; }; /** Basic abstraction of a named (system wide) mutex primitive @@ -143,7 +167,7 @@ public: ZENCORE_API bool Wait(int TimeoutMs = -1); ZENCORE_API void Terminate(int ExitCode); ZENCORE_API void Reset(); - inline [[nodiscard]] int Pid() const { return m_Pid; } + [[nodiscard]] inline int Pid() const { return m_Pid; } private: void* m_ProcessHandle = nullptr; @@ -174,6 +198,7 @@ private: ZENCORE_API bool IsProcessRunning(int pid); ZENCORE_API int GetCurrentProcessId(); +ZENCORE_API int GetCurrentThreadId(); ZENCORE_API void Sleep(int ms); diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h index f4e9ab65a..d25aa8059 100644 --- a/zencore/include/zencore/uid.h +++ b/zencore/include/zencore/uid.h @@ -74,7 +74,7 @@ struct Oid size_t operator()(const Oid& id) const { const size_t seed = id.OidBits[0]; - return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32); + return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32); } }; diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h index 0c40dd66b..d7f2ebfb7 100644 --- a/zencore/include/zencore/varint.h +++ b/zencore/include/zencore/varint.h @@ -2,6 +2,8 @@ #include "intmath.h" +#include <algorithm> + namespace zen { // Variable-Length Integer Encoding diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h index 68138566b..4149506e7 100644 --- a/zencore/include/zencore/windows.h +++ b/zencore/include/zencore/windows.h @@ -10,7 +10,12 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro #ifndef NOMINMAX # define NOMINMAX // We don't want your min/max macros #endif -#define WIN32_LEAN_AND_MEAN +#ifndef NOGDI +# define NOGDI // We don't want your GetObject define +#endif +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif #include <windows.h> #undef GetObject diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index d654770d0..195662715 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -29,6 +29,18 @@ # define ZEN_PLATFORM_MACOS 1 #endif +#if ZEN_PLATFORM_WINDOWS +# if !defined(NOMINMAX) +# define NOMINMAX // stops Windows.h from defining 'min/max' macros +# endif +# if !defined(NOGDI) +# define NOGDI +# endif +# if !defined(WIN32_LEAN_AND_MEAN) +# define WIN32_LEAN_AND_MEAN // cut-down what Windows.h defines +# endif +#endif + ////////////////////////////////////////////////////////////////////////// // Compiler // @@ -66,7 +78,11 @@ #ifndef ZEN_THIRD_PARTY_INCLUDES_START # if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_START __pragma(warning(push)) __pragma(warning(disable : 4668)) +# define ZEN_THIRD_PARTY_INCLUDES_START \ + __pragma(warning(push)) \ + __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ + __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ + __pragma(warning(disable : 4127)) # else # define ZEN_THIRD_PARTY_INCLUDES_START # endif @@ -80,6 +96,12 @@ # endif #endif +#if ZEN_COMPILER_MSC +# define ZEN_DEBUG_BREAK() do { __debugbreak(); } while (0) +#else +# define ZEN_DEBUG_BREAK() do { __builtin_trap(); } while (0) +#endif + ////////////////////////////////////////////////////////////////////////// // Architecture // @@ -110,6 +132,13 @@ #define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 +#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4 +# define ZEN_SIZEOF_WCHAR_T 4 +#else + static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size"); +# define ZEN_SIZEOF_WCHAR_T 2 +#endif + ////////////////////////////////////////////////////////////////////////// // Assert // @@ -229,9 +258,15 @@ ZENCORE_API void zencore_forcelinktests(); ////////////////////////////////////////////////////////////////////////// #if ZEN_COMPILER_MSC -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) # define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) -#else +#elif ZEN_COMPILER_GCC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC push_options" ) \ + _Pragma( "GCC optimize (\"O0\")" ) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC pop_options" ) +#elif ZEN_COMPILER_CLANG +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off") +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on") #endif // Set up optimization control macros, now that we have both the build settings and the platform macros diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 7077942bf..fd652f580 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -391,18 +391,22 @@ IoBufferBuilder::ReadFromFileMaybe(IoBuffer& InBuffer) DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(FileRef.FileHandle, OutBuffer.MutableData(), DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); +#else + int Fd = int(intptr_t(FileRef.FileHandle)); + int Result = pread(Fd, OutBuffer.MutableData(), size_t(FileRef.FileChunkSize), off_t(FileRef.FileChunkOffset)); + bool Success = (Result < 0); + + uint32_t dwNumberOfBytesRead = uint32_t(Result); +#endif if (!Success) { ThrowLastError("ReadFile failed in IoBufferBuilder::ReadFromFileMaybe"); } - ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); + ZEN_ASSERT(dwNumberOfBytesRead == FileRef.FileChunkSize); return OutBuffer; -#else -# error Needs implementation -#endif } else { @@ -417,14 +421,14 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) { uint64_t FileSize; #if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; - HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); if (FAILED(hRes)) { @@ -433,7 +437,7 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint DataFile.GetSize((ULONGLONG&)FileSize); #else - int Fd = open(FileName, O_RDONLY); + int Fd = open(FileName.c_str(), O_RDONLY); if (Fd < 0) { return {}; diff --git a/zencore/memory.cpp b/zencore/memory.cpp index c94829276..1345d6e69 100644 --- a/zencore/memory.cpp +++ b/zencore/memory.cpp @@ -1,5 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include <zencore/zencore.h> #include <zencore/intmath.h> #include <zencore/memory.h> #include <zencore/testing.h> diff --git a/zencore/stats.cpp b/zencore/stats.cpp index 0c0647999..1bb6f6de0 100644 --- a/zencore/stats.cpp +++ b/zencore/stats.cpp @@ -227,7 +227,7 @@ UniformSample::Snapshot() const uint64_t ValuesSize = Size(); std::vector<double> Values(ValuesSize); - for (int i = 0; i < ValuesSize; ++i) + for (int i = 0, n = int(ValuesSize); i < n; ++i) { Values[i] = double(m_Values[i]); } diff --git a/zencore/string.cpp b/zencore/string.cpp index b7670617f..35dfcfcf3 100644 --- a/zencore/string.cpp +++ b/zencore/string.cpp @@ -30,6 +30,17 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild } } +template<typename u32bit_iterator> +void +utf32to8_impl(u32bit_iterator StartIt, u32bit_iterator EndIt, ::zen::StringBuilderBase& OutString) +{ + for (; StartIt != EndIt; ++StartIt) + { + wchar_t cp = *StartIt; + OutString.AppendCodepoint(cp); + } +} + ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -76,7 +87,7 @@ FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch // Look for extension introducer ('.') - for (size_t i = PathLen - 1; i >= 0; --i) + for (int64_t i = PathLen - 1; i >= 0; --i) { if (Path[i] == '.') return Path.data() + i; @@ -164,26 +175,24 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) void WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, OutString); + WideToUtf8(std::wstring_view{Wstr}, OutString); } void WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, OutString); -} - -void -WideToUtf8(const std::u16string_view& Wstr, StringBuilderBase& OutString) -{ +#if ZEN_SIZEOF_WCHAR_T == 2 utf16to8_impl(begin(Wstr), end(Wstr), OutString); +#else + utf32to8_impl(begin(Wstr), end(Wstr), OutString); +#endif } std::string WideToUtf8(const wchar_t* Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, String); + WideToUtf8(std::wstring_view{Wstr}, String); return String.c_str(); } @@ -192,7 +201,7 @@ std::string WideToUtf8(const std::wstring_view Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, String); + WideToUtf8(std::wstring_view{Wstr.data(), Wstr.size()}, String); return String.c_str(); } @@ -942,6 +951,18 @@ TEST_CASE("string") CHECK_EQ(ToLower("TE%St"sv), "te%st"sv); } + SUBCASE("StrCaseCompare") + { + CHECK(StrCaseCompare("foo", "FoO") == 0); + CHECK(StrCaseCompare("Bar", "bAs") < 0); + CHECK(StrCaseCompare("bAr", "Bas") < 0); + CHECK(StrCaseCompare("BBr", "Bar") > 0); + CHECK(StrCaseCompare("Bbr", "BAr") > 0); + CHECK(StrCaseCompare("foo", "FoO", 3) == 0); + CHECK(StrCaseCompare("Bar", "bAs", 3) < 0); + CHECK(StrCaseCompare("BBr", "Bar", 2) > 0); + } + SUBCASE("ForEachStrTok") { const auto Tokens = "here,is,my,different,tokens"sv; diff --git a/zencore/thread.cpp b/zencore/thread.cpp index da711fe89..7f8042ae0 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -4,10 +4,21 @@ #include <zencore/except.h> #include <zencore/string.h> +#include <zencore/testing.h> #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> #elif ZEN_PLATFORM_LINUX +# include <chrono> +# include <condition_variable> +# include <mutex> + +# include <poll.h> +# include <pthread.h> +# include <signal.h> +# include <sys/socket.h> +# include <sys/un.h> +# include <time.h> # include <unistd.h> #endif @@ -69,6 +80,8 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) std::string ThreadNameZ{ThreadName}; SetNameInternal(GetCurrentThreadId(), ThreadNameZ.c_str()); #else + std::string ThreadNameZ{ThreadName}; + pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); #endif } // namespace zen @@ -98,40 +111,80 @@ RwLock::ReleaseExclusive() ////////////////////////////////////////////////////////////////////////// -#if ZEN_PLATFORM_WINDOWS +#if !ZEN_PLATFORM_WINDOWS +struct EventInner +{ + std::mutex Mutex; + std::condition_variable CondVar; + bool volatile bSet = false; +}; +#endif // !ZEN_PLATFORM_WINDOWS Event::Event() { - m_EventHandle = CreateEvent(nullptr, true, false, nullptr); + bool bManualReset = true; + bool bInitialState = false; + +#if ZEN_PLATFORM_WINDOWS + m_EventHandle = CreateEvent(nullptr, bManualReset, bInitialState, nullptr); +#else + ZEN_UNUSED(bManualReset); + auto* Inner = new EventInner(); + Inner->bSet = bInitialState; + m_EventHandle = Inner; +#endif } Event::~Event() { - CloseHandle(m_EventHandle); + Close(); } void Event::Set() { +#if ZEN_PLATFORM_WINDOWS SetEvent(m_EventHandle); +#else + auto* Inner = (EventInner*)m_EventHandle; + { + std::unique_lock Lock(Inner->Mutex); + Inner->bSet = true; + } + Inner->CondVar.notify_all(); +#endif } void Event::Reset() { +#if ZEN_PLATFORM_WINDOWS ResetEvent(m_EventHandle); +#else + auto* Inner = (EventInner*)m_EventHandle; + { + std::unique_lock Lock(Inner->Mutex); + Inner->bSet = false; + } +#endif } void Event::Close() { +#if ZEN_PLATFORM_WINDOWS CloseHandle(m_EventHandle); +#else + auto* Inner = (EventInner*)m_EventHandle; + delete Inner; +#endif m_EventHandle = nullptr; } bool Event::Wait(int TimeoutMs) { +#if ZEN_PLATFORM_WINDOWS using namespace std::literals; const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; @@ -144,12 +197,50 @@ Event::Wait(int TimeoutMs) } return (Result == WAIT_OBJECT_0); +#else + auto* Inner = (EventInner*)m_EventHandle; + + if (TimeoutMs >= 0) + { + std::unique_lock Lock(Inner->Mutex); + + if (Inner->bSet) + { + return true; + } + + return Inner->CondVar.wait_for( + Lock, + std::chrono::milliseconds(TimeoutMs), + [&] { return Inner->bSet; } + ); + } + + std::unique_lock Lock(Inner->Mutex); + + if (!Inner->bSet) + { + Inner->CondVar.wait(Lock, [&] { return Inner->bSet; }); + } + + return true; +#endif } ////////////////////////////////////////////////////////////////////////// -NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr) +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MACOS +struct NamedEventPosix +{ + int SocketFd = 0; + sockaddr_un SocketAddr = {}; + bool bBound = false; +}; +#endif + +NamedEvent::NamedEvent(std::string_view EventName) { +#if ZEN_PLATFORM_WINDOWS using namespace std::literals; ExtendableStringBuilder<64> Name; @@ -157,19 +248,117 @@ NamedEvent::NamedEvent(std::u8string_view EventName) : Event(nullptr) Name << EventName; m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str()); +#else + int SocketFd = socket(AF_UNIX, SOCK_DGRAM, SOCK_CLOEXEC); + if (SocketFd < 0) + { + ThrowLastError("Failed to create IPC socket"); + } + + auto* Inner = new NamedEventPosix(); + Inner->SocketFd = SocketFd; + + char* PathPtr = Inner->SocketAddr.sun_path; + size_t PathLen = sizeof(Inner->SocketAddr.sun_path) - 1; // -1 for null-term +# if ZEN_PLATFORM_LINUX + PathPtr[0] = '\0'; // make the domain socket... + PathPtr += 1; // ...use the abstract namespace + PathLen -= 1; +# endif + EventName.copy(PathPtr, PathLen); + + m_EventHandle = Inner; +#endif } -NamedEvent::NamedEvent(std::string_view EventName) : Event(nullptr) +NamedEvent::~NamedEvent() { - using namespace std::literals; + Close(); +} - ExtendableStringBuilder<64> Name; - Name << "Local\\"sv; - Name << EventName; +void NamedEvent::Close() +{ + if (m_EventHandle == nullptr) + { + return; + } - m_EventHandle = CreateEventA(nullptr, true, false, Name.c_str()); +#if ZEN_PLATFORM_WINDOWS + CloseHandle(m_EventHandle); +#else + auto* Inner = (NamedEventPosix*)m_EventHandle; + close(Inner->SocketFd); + delete Inner; +#endif + + m_EventHandle = nullptr; +} + + +void NamedEvent::Set() +{ +#if ZEN_PLATFORM_WINDOWS + SetEvent(m_EventHandle); +#else + auto* Inner = (NamedEventPosix*)m_EventHandle; + + uint8_t OneByte = 0x49; + sendto( + Inner->SocketFd, + &OneByte, sizeof(OneByte), + 0, (sockaddr*)&Inner->SocketAddr, sizeof(Inner->SocketAddr) + ); +#endif +} + +bool NamedEvent::Wait(int TimeoutMs) +{ +#if ZEN_PLATFORM_WINDOWS + const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; + + DWORD Result = WaitForSingleObject(m_EventHandle, Timeout); + + if (Result == WAIT_FAILED) + { + using namespace std::literals; + zen::ThrowLastError("Event wait failed"sv); + } + + return (Result == WAIT_OBJECT_0); +#else + auto* Inner = (NamedEventPosix*)m_EventHandle; + int SocketFd = Inner->SocketFd; + + int Result; + + if (!Inner->bBound) + { + Result = bind(SocketFd, (sockaddr*)&(Inner->SocketAddr), sizeof(Inner->SocketAddr)); + if (!Result) + { + zen::ThrowLastError("Bind IPC socket failed"); + } + Inner->bBound = true; + } + + pollfd PollFd = { SocketFd, POLLIN }; + Result = poll(&PollFd, 1, TimeoutMs); + if (Result > 0) + { + uint8_t OneByte; + Result = recv(SocketFd, &OneByte, sizeof(OneByte), 0); + + return true; + } + + return false; +#endif } +////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS + NamedMutex::~NamedMutex() { if (m_MutexHandle) @@ -217,20 +406,26 @@ NamedMutex::Exists(std::string_view MutexName) #endif // ZEN_PLATFORM_WINDOWS -#if ZEN_PLATFORM_WINDOWS - ////////////////////////////////////////////////////////////////////////// ProcessHandle::ProcessHandle() = default; +#if ZEN_PLATFORM_WINDOWS void ProcessHandle::Initialize(void* ProcessHandle) { ZEN_ASSERT(m_ProcessHandle == nullptr); + + if (ProcessHandle == INVALID_HANDLE_VALUE) + { + ProcessHandle = nullptr; + } + // TODO: perform some debug verification here to verify it's a valid handle? m_ProcessHandle = ProcessHandle; m_Pid = GetProcessId(m_ProcessHandle); } +#endif // ZEN_PLATFORM_WINDOWS ProcessHandle::~ProcessHandle() { @@ -241,12 +436,21 @@ void ProcessHandle::Initialize(int Pid) { ZEN_ASSERT(m_ProcessHandle == nullptr); - m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid); - using namespace fmt::literals; +#if ZEN_PLATFORM_WINDOWS + m_ProcessHandle = OpenProcess(PROCESS_QUERY_INFORMATION | SYNCHRONIZE, FALSE, Pid); +#elif ZEN_PLATFORM_LINUX + if (Pid > 0) + { + m_ProcessHandle = (void*)(intptr_t(Pid)); + } +#else +# error Check process control on this platform +#endif if (!m_ProcessHandle) { + using namespace fmt::literals; ThrowLastError("ProcessHandle::Initialize(pid: {}) failed"_format(Pid)); } @@ -256,29 +460,51 @@ ProcessHandle::Initialize(int Pid) bool ProcessHandle::IsRunning() const { + bool bActive = false; + +#if ZEN_PLATFORM_WINDOWS DWORD ExitCode = 0; GetExitCodeProcess(m_ProcessHandle, &ExitCode); + bActive = (ExitCode == STILL_ACTIVE); +#elif ZEN_PLATFORM_LINUX + StringBuilder<64> ProcPath; + ProcPath << "/proc/" << m_Pid; + bActive = (access(ProcPath.c_str(), F_OK) != 0); +#else +# error Check process control on this platform +#endif - return ExitCode == STILL_ACTIVE; + return bActive; } bool ProcessHandle::IsValid() const { - return (m_ProcessHandle != nullptr) && (m_ProcessHandle != INVALID_HANDLE_VALUE); + return (m_ProcessHandle != nullptr); } void ProcessHandle::Terminate(int ExitCode) { - if (IsRunning()) + if (!IsRunning()) { - TerminateProcess(m_ProcessHandle, ExitCode); + return; } + bool bSuccess = false; + +#if ZEN_PLATFORM_WINDOWS + TerminateProcess(m_ProcessHandle, ExitCode); DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, INFINITE); + bSuccess = (WaitResult != WAIT_OBJECT_0); +#elif ZEN_PLATFORM_LINUX + ZEN_UNUSED(ExitCode); + bSuccess = (kill(m_Pid, SIGKILL) == 0); +#else +# error Check kill() on this platform +#endif - if (WaitResult != WAIT_OBJECT_0) + if (!bSuccess) { // What might go wrong here, and what is meaningful to act on? } @@ -289,14 +515,20 @@ ProcessHandle::Reset() { if (IsValid()) { +#if ZEN_PLATFORM_WINDOWS CloseHandle(m_ProcessHandle); +#endif m_ProcessHandle = nullptr; + m_Pid = 0; } } bool ProcessHandle::Wait(int TimeoutMs) { + using namespace std::literals; + +#if ZEN_PLATFORM_WINDOWS const DWORD Timeout = (TimeoutMs < 0) ? INFINITE : TimeoutMs; const DWORD WaitResult = WaitForSingleObject(m_ProcessHandle, Timeout); @@ -310,15 +542,36 @@ ProcessHandle::Wait(int TimeoutMs) return false; case WAIT_FAILED: - // What might go wrong here, and what is meaningful to act on? - using namespace std::literals; - ThrowLastError("Process::Wait failed"sv); + break; } +#elif ZEN_PLATFORM_LINUX + const int SleepMs = 20; + timespec SleepTime = { 0, SleepMs * 1000 * 1000 }; + for (int i = 0; ; i += SleepMs) + { + if (i >= TimeoutMs) + { + return false; + } - return false; -} + if (kill(m_Pid, 0) < 0) + { + if (zen::GetLastError() == ESRCH) + { + return true; + } + break; + } -#endif // ZEN_PLATFORM_WINDOWS + nanosleep(&SleepTime, nullptr); + } +#else +# error Check kill() on this platform +#endif + + // What might go wrong here, and what is meaningful to act on? + ThrowLastError("Process::Wait failed"sv); +} ////////////////////////////////////////////////////////////////////////// @@ -418,7 +671,9 @@ IsProcessRunning(int pid) return true; #else - ZEN_NOT_IMPLEMENTED(); + char Buffer[64]; + sprintf(Buffer, "/proc/%d", pid); + return access(Buffer, F_OK) == 0; #endif } @@ -428,7 +683,17 @@ GetCurrentProcessId() #if ZEN_PLATFORM_WINDOWS return ::GetCurrentProcessId(); #else - return getpid(); + return int(getpid()); +#endif +} + +int +GetCurrentThreadId() +{ +#if ZEN_PLATFORM_WINDOWS + return ::GetCurrentThreadId(); +#else + return int(gettid()); #endif } @@ -447,9 +712,20 @@ Sleep(int ms) // Testing related code follows... // +#if ZEN_WITH_TESTS + void thread_forcelink() { } +TEST_CASE("Thread") +{ + int Pid = GetCurrentProcessId(); + CHECK(Pid > 0); + CHECK(IsProcessRunning(Pid)); +} + +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/zencore/xmake.lua b/zencore/xmake.lua index d26a9f922..5923b99c3 100644 --- a/zencore/xmake.lua +++ b/zencore/xmake.lua @@ -2,8 +2,17 @@ target('zencore') set_kind("static") add_files("**.cpp") add_includedirs("include", {public=true}) - add_includedirs("..\\thirdparty\\utfcpp\\source") - add_linkdirs("$(projectdir)/thirdparty/BLAKE3/lib/Win64", "$(projectdir)/thirdparty/Oodle/lib/Win64") + add_includedirs("$(projectdir)/thirdparty/utfcpp/source") + if is_os("windows") then + add_linkdirs("$(projectdir)/thirdparty/BLAKE3/lib/Win64") + add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Win64") + elseif is_os("linux") then + add_linkdirs("$(projectdir)/thirdparty/BLAKE3/lib/Linux_x64") + add_linkdirs("$(projectdir)/thirdparty/Oodle/lib/Linux_x64") + add_links("blake3") + add_links("oo2corelinux64") + add_syslinks("pthread") + end add_packages( "vcpkg::spdlog", "vcpkg::fmt", @@ -14,5 +23,21 @@ target('zencore') "vcpkg::cpr", "vcpkg::curl", -- required by cpr "vcpkg::zlib", -- required by curl + "vcpkg::openssl", -- required by curl "vcpkg::xxhash", "vcpkg::gsl-lite") + + if is_plat("linux") then + -- The 'vcpkg::openssl' package is two libraries; ssl and crypto, with + -- ssl being dependent on symbols in crypto. When GCC-like linkers read + -- object files from their command line, those object files only resolve + -- symbols of objects previously encountered. Thus crypto must appear + -- after ssl so it can fill out ssl's unresolved symbol table. Xmake's + -- vcpkg support is basic and works by parsing .list files. Openssl's + -- archives are listed alphabetically causing crypto to be _before_ ssl + -- and resulting in link errors. The links are restated here to force + -- xmake to use the correct order, and "syslinks" is used to force the + -- arguments to the end of the line (otherwise they can appear before + -- curl and cause more errors). + add_syslinks("ssl", "crypto") + end diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp index ea84e7e87..d94f6af2e 100644 --- a/zenhttp/httpasio.cpp +++ b/zenhttp/httpasio.cpp @@ -9,7 +9,9 @@ #include <memory_resource> ZEN_THIRD_PARTY_INCLUDES_START -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif #include <http_parser.h> #include <asio.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -393,7 +395,7 @@ HttpServerConnection::EnqueueRead() ZEN_TRACE("read: conn#:{} seq#:{} t:{} bytes:{}", m_ConnectionId, m_RequestCounter.load(std::memory_order_relaxed), - GetCurrentThreadId(), + zen::GetCurrentThreadId(), ByteCount); OnDataReceived(Ec, ByteCount); @@ -1017,7 +1019,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode) m_Response.reset(new HttpResponse(HttpContentType::kBinary)); std::array<IoBuffer, 0> Empty; - m_Response->InitializeForPayload((UINT16)ResponseCode, Empty); + m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty); } void @@ -1026,7 +1028,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_ASSERT(!m_Response); m_Response.reset(new HttpResponse(ContentType)); - m_Response->InitializeForPayload((UINT16)ResponseCode, Blobs); + m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs); } void @@ -1157,6 +1159,13 @@ HttpAsioServer::Run(bool IsInteractive) { const bool TestMode = !IsInteractive; + int WaitTimeout = -1; + if (!TestMode) + { + WaitTimeout = 1000; + } + +#if ZEN_PLATFORM_WINDOWS if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (asio HTTP). Press ESC or Q to quit"); @@ -1164,13 +1173,6 @@ HttpAsioServer::Run(bool IsInteractive) do { - int WaitTimeout = -1; - - if (!TestMode) - { - WaitTimeout = 1000; - } - if (!TestMode && _kbhit() != 0) { char c = (char)_getch(); @@ -1183,6 +1185,18 @@ HttpAsioServer::Run(bool IsInteractive) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); +#else + if (TestMode == false) + { + zen::logging::ConsoleLog().info("Zen Server running (asio HTTP). Ctrl-C to quit"); + } + + do + { + m_ShutdownEvent.Wait(WaitTimeout); + } + while (!IsApplicationExitRequested()); +#endif } void diff --git a/zenhttp/httpclient.cpp b/zenhttp/httpclient.cpp index 6e915e613..e6813d407 100644 --- a/zenhttp/httpclient.cpp +++ b/zenhttp/httpclient.cpp @@ -22,7 +22,7 @@ using namespace std::literals; HttpClient::Response FromCprResponse(cpr::Response& InResponse) { - return {.StatusCode = InResponse.status_code}; + return {.StatusCode = int(InResponse.status_code)}; } ////////////////////////////////////////////////////////////////////////// @@ -130,7 +130,7 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package) ResponseBuffer.SetContentType(ContentType); } - return {.StatusCode = FilterResponse.status_code, .ResponsePayload = ResponseBuffer}; + return {.StatusCode = int(FilterResponse.status_code), .ResponsePayload = ResponseBuffer}; } HttpClient::Response diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp index e49051ac5..1a60e211c 100644 --- a/zenhttp/httpnull.cpp +++ b/zenhttp/httpnull.cpp @@ -2,9 +2,12 @@ #include "httpnull.h" -#include <conio.h> #include <zencore/logging.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif + namespace zen { HttpNullServer::HttpNullServer() @@ -32,6 +35,13 @@ HttpNullServer::Run(bool IsInteractiveSession) { const bool TestMode = !IsInteractiveSession; + int WaitTimeout = -1; + if (!TestMode) + { + WaitTimeout = 1000; + } + +#if ZEN_PLATFORM_WINDOWS if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); @@ -39,13 +49,6 @@ HttpNullServer::Run(bool IsInteractiveSession) do { - int WaitTimeout = -1; - - if (!TestMode) - { - WaitTimeout = 1000; - } - if (!TestMode && _kbhit() != 0) { char c = (char)_getch(); @@ -58,6 +61,18 @@ HttpNullServer::Run(bool IsInteractiveSession) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); +#else + if (TestMode == false) + { + zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Ctrl-C to quit"); + } + + do + { + m_ShutdownEvent.Wait(WaitTimeout); + } + while (!IsApplicationExitRequested()); +#endif } void diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index f40836c4a..7b729cf0e 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -17,8 +17,6 @@ #include <zencore/thread.h> #include <zenhttp/httpshared.h> -#include <conio.h> -#include <new.h> #include <charconv> #include <mutex> #include <span> diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index c703409af..ab1463559 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -109,7 +109,7 @@ FormatPackageMessage(const CbPackage& Data) } } - return std::move(ResponseBuffers); + return ResponseBuffers; } CbPackage diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index cdf9e0a39..242954912 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -1221,7 +1221,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& const HTTP_REQUEST* HttpRequestPtr = Tx.HttpRequest(); const int PrefixLength = Service.UriPrefixLength(); - const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t); + const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(wchar_t); HttpContentType AcceptContentType = HttpContentType::kUnknownContentType; @@ -1231,7 +1231,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& // with utf8. This is overhead which I'd prefer to avoid but for now we just have // to live with it - WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, + WideToUtf8({(wchar_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, m_UriUtf8); std::string_view UriSuffix8{m_UriUtf8}; @@ -1268,7 +1268,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& { --QueryStringLength; // We skip the leading question mark - WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryStringUtf8); + WideToUtf8({(wchar_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(wchar_t)}, m_QueryStringUtf8); } else { diff --git a/zenhttp/include/zenhttp/httpclient.h b/zenhttp/include/zenhttp/httpclient.h index 9ece86111..8316a9b9f 100644 --- a/zenhttp/include/zenhttp/httpclient.h +++ b/zenhttp/include/zenhttp/httpclient.h @@ -8,8 +8,6 @@ #include <zencore/uid.h> #include <zenhttp/httpcommon.h> -#include <zencore/windows.h> - ZEN_THIRD_PARTY_INCLUDES_START #include <cpr/cpr.h> ZEN_THIRD_PARTY_INCLUDES_END diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h index 93ba452c7..614cdc2ac 100644 --- a/zenhttp/include/zenhttp/httpserver.h +++ b/zenhttp/include/zenhttp/httpserver.h @@ -46,7 +46,7 @@ public: if (Key.size() == ParamName.size()) { - if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size())) + if (0 == StrCaseCompare(Key.data(), ParamName.data(), Key.size())) { return Kv.second; } diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h index 2e728577d..dc660e45d 100644 --- a/zenhttp/include/zenhttp/httpshared.h +++ b/zenhttp/include/zenhttp/httpshared.h @@ -36,7 +36,7 @@ struct CbPackageHeader static_assert(sizeof(CbPackageHeader) == 16); -static constinit uint32_t kCbPkgMagic = 0xaa77aacc; +enum : uint32_t { kCbPkgMagic = 0xaa77aacc }; struct CbAttachmentEntry { diff --git a/zenhttp/workthreadpool.h b/zenhttp/workthreadpool.h index 6581cc08f..834339d50 100644 --- a/zenhttp/workthreadpool.h +++ b/zenhttp/workthreadpool.h @@ -6,7 +6,6 @@ #include <zencore/blockingqueue.h> #include <zencore/refcount.h> -#include <zencore/windows.h> #include <exception> #include <functional> diff --git a/zenhttp/xmake.lua b/zenhttp/xmake.lua index fff4fb526..a94069c17 100644 --- a/zenhttp/xmake.lua +++ b/zenhttp/xmake.lua @@ -3,5 +3,8 @@ target('zenhttp') add_files("**.cpp") add_includedirs("include", {public=true}) add_deps("zencore") - add_packages("vcpkg::gsl-lite") + add_packages( + "vcpkg::gsl-lite", + "vcpkg::http-parser" + ) add_options("httpsys")
\ No newline at end of file diff --git a/zenserver-test/projectclient.cpp b/zenserver-test/projectclient.cpp index 2700ae9da..2de9b6ecf 100644 --- a/zenserver-test/projectclient.cpp +++ b/zenserver-test/projectclient.cpp @@ -2,6 +2,8 @@ #include "projectclient.h" +#if 0 + #include <zencore/compactbinary.h> #include <zencore/logging.h> #include <zencore/sharedbuffer.h> @@ -11,7 +13,9 @@ #include <asio.hpp> #include <gsl/gsl-lite.hpp> -#include <atlbase.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlbase.h> +#endif namespace zen { @@ -158,3 +162,5 @@ LocalProjectClient::MessageTransaction(CbObject Request) } } // namespace zen + +#endif // 0 diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index 8f38cc1be..2fd8bdfcb 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -41,16 +41,19 @@ ZEN_THIRD_PARTY_INCLUDES_START #undef GetObject ZEN_THIRD_PARTY_INCLUDES_END -#include <ppl.h> #include <atomic> #include <filesystem> #include <map> #include <random> #include <span> +#include <thread> #include <unordered_map> -#include <atlbase.h> -#include <process.h> +#if ZEN_PLATFORM_WINDOWS +# include <ppl.h> +# include <atlbase.h> +# include <process.h> +#endif #include <asio.hpp> @@ -69,6 +72,25 @@ ZEN_THIRD_PARTY_INCLUDES_END using namespace fmt::literals; using namespace std::literals; +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MACOS +struct Concurrency +{ + template <typename... T> + static void parallel_invoke(T&&... t) + { + constexpr size_t NumTs = sizeof...(t); + std::thread Threads[NumTs] = { + std::thread(std::forward<T>(t))..., + }; + + for (std::thread& Thread : Threads) + { + Thread.join(); + } + } +}; +#endif + /* ___ ___ _________ _________ ________ ________ ___ ___ _______ ________ _________ @@ -341,7 +363,7 @@ HttpConnectionPool::GetConnection() std::unique_ptr<HttpClientConnection> Connection{m_AvailableConnections.back()}; m_AvailableConnections.pop_back(); - return std::move(Connection); + return Connection; } void @@ -740,7 +762,7 @@ TEST_CASE("default.single") auto IssueTestRequests = [&] { const uint64_t BatchNo = BatchCounter.fetch_add(1); - const DWORD ThreadId = GetCurrentThreadId(); + const int ThreadId = zen::GetCurrentThreadId(); ZEN_INFO("query batch {} started (thread {})", BatchNo, ThreadId); cpr::Session cli; @@ -754,22 +776,8 @@ TEST_CASE("default.single") ZEN_INFO("query batch {} ended (thread {})", BatchNo, ThreadId); }; - auto fun10 = [&] { - Concurrency::parallel_invoke(IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests, - IssueTestRequests); - }; - zen::Stopwatch timer; - // Concurrency::parallel_invoke(fun10, fun10, fun, fun, fun, fun, fun, fun, fun, fun); Concurrency::parallel_invoke(IssueTestRequests, IssueTestRequests, IssueTestRequests, @@ -808,7 +816,7 @@ TEST_CASE("multi.basic") auto IssueTestRequests = [&](int PortNumber) { const uint64_t BatchNo = BatchCounter.fetch_add(1); - const DWORD ThreadId = GetCurrentThreadId(); + const int ThreadId = zen::GetCurrentThreadId(); ZEN_INFO("query batch {} started (thread {}) for port {}", BatchNo, ThreadId, PortNumber); @@ -1744,6 +1752,8 @@ TEST_CASE("zcache.policy") zen::CbPackage Package; const bool Ok = Package.TryLoad(Body); + CHECK(Ok); + CbObject CacheRecord = Package.GetObject(); std::vector<IoHash> AttachmentKeys; @@ -1763,6 +1773,7 @@ TEST_CASE("zcache.policy") zen::CbPackage Package; const bool Ok = Package.TryLoad(Body); + CHECK(Ok); CHECK(Package.GetAttachments().size() != 0); } } @@ -1802,6 +1813,8 @@ TEST_CASE("zcache.policy") zen::CbPackage Package; const bool Ok = Package.TryLoad(Body); + CHECK(Ok); + CbObject CacheRecord = Package.GetObject(); std::vector<IoHash> AttachmentKeys; @@ -1821,6 +1834,7 @@ TEST_CASE("zcache.policy") zen::CbPackage Package; const bool Ok = Package.TryLoad(Body); + CHECK(Ok); CHECK(Package.GetAttachments().size() != 0); } } @@ -2221,7 +2235,7 @@ struct RemoteExecutionRequest for (const auto& Kv : m_Visit.m_Files) { PrepReq.BeginObject(); - PrepReq << "file" << zen::WideToUtf8(Kv.first) << "size" << Kv.second.Size << "hash" << Kv.second.Hash; + PrepReq << "file" << zen::ToUtf8(Kv.first) << "size" << Kv.second.Size << "hash" << Kv.second.Hash; PrepReq.EndObject(); } PrepReq.EndArray(); @@ -2244,7 +2258,7 @@ struct RemoteExecutionRequest if (auto It = m_Visit.m_HashToFile.find(NeedHash); It != m_Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url(m_CasUri), cpr::Body((const char*)FileData.Data(), FileData.Size())); @@ -2282,7 +2296,7 @@ private: Visitor(const std::filesystem::path& RootPath) : m_RootPath(RootPath) {} - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& FileName, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& FileName, uint64_t FileSize) override { std::filesystem::path FullPath = Parent / FileName; @@ -2290,8 +2304,8 @@ private: zen::ScanFile(FullPath, 64 * 1024, [&](const void* Data, size_t Size) { Ios.Append(Data, Size); }); zen::IoHash Hash = Ios.GetHash(); - std::wstring RelativePath = FullPath.lexically_relative(m_RootPath).native(); - // ZEN_INFO("File: {:32} => {} ({})", zen::WideToUtf8(RelativePath), Hash, FileSize); + auto RelativePath = FullPath.lexically_relative(m_RootPath).native(); + // ZEN_INFO("File: {:32} => {} ({})", zen::ToUtf8(RelativePath), Hash, FileSize); FileEntry& Entry = m_Files[RelativePath]; Entry.Hash = Hash; @@ -2300,11 +2314,11 @@ private: m_HashToFile[Hash] = FullPath; } - virtual bool VisitDirectory(const std::filesystem::path& Parent, const std::wstring_view& DirectoryName) override + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName) override { std::filesystem::path FullPath = Parent / DirectoryName; - if (DirectoryName.starts_with(L".")) + if (DirectoryName.starts_with('.')) { return false; } @@ -2318,7 +2332,7 @@ private: zen::IoHash Hash; }; - std::map<std::wstring, FileEntry> m_Files; + std::map<std::filesystem::path::string_type, FileEntry> m_Files; std::unordered_map<zen::IoHash, std::filesystem::path, zen::IoHash::Hasher> m_HashToFile; }; @@ -2333,6 +2347,7 @@ private: TEST_CASE("exec.basic") { +#if ZEN_WITH_COMPUTE_SERVICES using namespace std::literals; std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); @@ -2363,6 +2378,7 @@ TEST_CASE("exec.basic") CHECK(Result["exitcode"].AsInt32(-1) == 1); } +#endif // ZEN_WITH_COMPUTE_SERVICES } TEST_CASE("mesh.basic") diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp index 07211cbeb..44406fa70 100644 --- a/zenserver/admin/admin.cpp +++ b/zenserver/admin/admin.cpp @@ -1,7 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#pragma once - #include "admin.h" #include <zencore/compactbinarybuilder.h> diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 726bd7cdb..6fba7648f 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -10,7 +10,7 @@ #include <zencore/stream.h> #include <zencore/timer.h> #include <zenhttp/httpserver.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenutil/cache/cache.h> //#include "cachekey.h" @@ -155,7 +155,7 @@ void HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, std::string_view Bucket) { ZEN_UNUSED(Request, Bucket); - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -178,13 +178,16 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, return Request.WriteResponse(HttpResponseCode::NotFound); } break; + + default: + break; } } void HttpStructuredCacheService::HandleCacheRecordRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -445,7 +448,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kBinary, .CacheKey = {Ref.BucketSegment, Ref.HashKey}}); } Request.WriteResponse(HttpResponseCode::Created); @@ -489,7 +492,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbObject, .CacheKey = {Ref.BucketSegment, Ref.HashKey}, .PayloadIds = std::move(ValidAttachments)}); } @@ -571,9 +574,9 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request if (StoreUpstream && !IsPartialRecord) { ZEN_ASSERT(m_UpstreamCache); - auto Result = m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, - .CacheKey = {Ref.BucketSegment, Ref.HashKey}, - .PayloadIds = std::move(ValidAttachments)}); + m_UpstreamCache->EnqueueUpstream({.Type = ZenContentType::kCbPackage, + .CacheKey = {Ref.BucketSegment, Ref.HashKey}, + .PayloadIds = std::move(ValidAttachments)}); } Request.WriteResponse(HttpResponseCode::Created); @@ -587,7 +590,7 @@ HttpStructuredCacheService::HandlePutCacheRecord(zen::HttpServerRequest& Request void HttpStructuredCacheService::HandleCachePayloadRequest(HttpServerRequest& Request, const CacheRef& Ref, CachePolicy Policy) { - switch (auto Verb = Request.RequestVerb()) + switch (Request.RequestVerb()) { using enum HttpVerb; @@ -620,7 +623,7 @@ HttpStructuredCacheService::HandleGetCachePayload(zen::HttpServerRequest& Reques { Payload = UpstreamResult.Value; IoHash ChunkHash = IoHash::HashBuffer(Payload); - CasStore::InsertResult Result = m_CasStore.InsertChunk(Payload, ChunkHash); + /*CasStore::InsertResult Result = */ m_CasStore.InsertChunk(Payload, ChunkHash); InUpstreamCache = true; m_CidStore.AddCompressedCid(Ref.PayloadId, ChunkHash); diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index 8b9ce8ff9..2b43cddeb 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -3,7 +3,6 @@ #include "structuredcachestore.h" #include <zencore/except.h> -#include <zencore/windows.h> #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> @@ -23,6 +22,10 @@ #include <zenstore/cidstore.h> #include <zenstore/gc.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <concepts> #include <filesystem> #include <ranges> @@ -277,6 +280,9 @@ ZenCacheMemoryLayer::CacheBucket::GarbageCollect(GcContext& GcCtx) case ZenContentType::kBinary: break; + + default: + break; } } } @@ -471,17 +477,17 @@ ZenCacheDiskLayer::CacheBucket::OpenOrCreate(std::filesystem::path BucketDir, bo } void -ZenCacheDiskLayer::CacheBucket::BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey) +ZenCacheDiskLayer::CacheBucket::BuildPath(PathBuilderBase& Path, const IoHash& HashKey) { char HexString[sizeof(HashKey.Hash) * 2]; ToHexBytes(HashKey.Hash, sizeof HashKey.Hash, HexString); - Path.Append(m_BucketDir.c_str()); + Path.Append(m_BucketDir); Path.Append(L"/blob/"); Path.AppendAsciiRange(HexString, HexString + 3); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 3, HexString + 5); - Path.Append(L"/"); + Path.AppendSeparator(); Path.AppendAsciiRange(HexString + 5, HexString + sizeof(HexString)); } @@ -502,12 +508,12 @@ ZenCacheDiskLayer::CacheBucket::GetInlineCacheValue(const DiskLocation& Loc, Zen bool ZenCacheDiskLayer::CacheBucket::GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue) { - WideStringBuilder<128> DataFilePath; + PathBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); RwLock::SharedLockScope ValueLock(LockForHash(HashKey)); - if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.c_str())) + if (IoBuffer Data = IoBufferBuilder::MakeFromFile(DataFilePath.ToPath())) { OutValue.Value = Data; OutValue.Value.SetContentType(Loc.GetContentType()); @@ -708,7 +714,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c { RwLock::ExclusiveLockScope ValueLock(LockForHash(HashKey)); - WideStringBuilder<128> DataFilePath; + PathBuilder<128> DataFilePath; BuildPath(DataFilePath, HashKey); TemporaryFile DataFile; @@ -730,7 +736,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c // Move file into place (atomically) - std::filesystem::path FsPath{DataFilePath.c_str()}; + std::filesystem::path FsPath{DataFilePath.ToPath()}; DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -740,7 +746,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c do { - std::filesystem::path ParentPath = std::filesystem::path(DataFilePath.c_str()).parent_path(); + std::filesystem::path ParentPath = FsPath.parent_path(); CreateDirectories(ParentPath); DataFile.MoveTemporaryIntoPlace(FsPath, Ec); @@ -765,7 +771,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c if (Ec) { - throw std::system_error(Ec, "Failed to finalize file '{}'"_format(WideToUtf8(DataFilePath))); + throw std::system_error(Ec, "Failed to finalize file '{}'"_format(DataFilePath.ToUtf8())); } } @@ -907,11 +913,11 @@ ZenCacheDiskLayer::DiscoverBuckets() virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, const path_view& DirectoryName) override { - Dirs.push_back(std::wstring(DirectoryName)); + Dirs.push_back((decltype(Dirs)::value_type)(DirectoryName)); return false; } - std::vector<std::wstring> Dirs; + std::vector<std::filesystem::path::string_type> Dirs; } Visit; Traversal.TraverseFileSystem(m_RootDir, Visit); @@ -920,11 +926,15 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock::ExclusiveLockScope _(m_Lock); - for (const std::wstring& BucketName : Visit.Dirs) + for (const auto& BucketName : Visit.Dirs) { // New bucket needs to be created +#if ZEN_PLATFORM_WINDOWS std::string BucketName8 = WideToUtf8(BucketName); +#else + const auto& BucketName8 = BucketName; +#endif if (auto It = m_Buckets.find(BucketName8); It != m_Buckets.end()) { diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 0dfcbc5ca..e3013b53f 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -11,10 +11,9 @@ #include <zenstore/cas.h> #include <zenstore/caslog.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <compare> #include <filesystem> @@ -22,7 +21,7 @@ namespace zen { -class WideStringBuilderBase; +class PathBuilderBase; class CasStore; /****************************************************************************** @@ -187,7 +186,7 @@ private: tsl::robin_map<IoHash, DiskLocation, IoHash::Hasher> m_Index; uint64_t m_WriteCursor = 0; - void BuildPath(WideStringBuilderBase& Path, const IoHash& HashKey); + void BuildPath(PathBuilderBase& Path, const IoHash& HashKey); void PutStandaloneCacheValue(const IoHash& HashKey, const ZenCacheValue& Value); bool GetStandaloneCacheValue(const DiskLocation& Loc, const IoHash& HashKey, ZenCacheValue& OutValue); bool GetInlineCacheValue(const DiskLocation& Loc, ZenCacheValue& OutValue); diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 88525bd36..872a40df8 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -51,6 +51,9 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) return ServerRequest.WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 053c262c2..4a92b9968 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -2,6 +2,8 @@ #include "apply.h" +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/compactbinarypackage.h> @@ -12,20 +14,18 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/scopeguard.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/cidstore.h> -#if ZEN_PLATFORM_WINDOWS +#include <zencore/windows.h> ZEN_THIRD_PARTY_INCLUDES_START -# include <AccCtrl.h> -# include <AclAPI.h> -# include <sddl.h> -# include <UserEnv.h> -# pragma comment(lib, "UserEnv.lib") -# include <atlbase.h> +#include <AccCtrl.h> +#include <AclAPI.h> +#include <sddl.h> +#include <UserEnv.h> +#pragma comment(lib, "UserEnv.lib") +#include <atlbase.h> ZEN_THIRD_PARTY_INCLUDES_END -#endif #include <filesystem> #include <span> @@ -134,6 +134,8 @@ BasicFunctionJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedFunctionJob { SandboxedFunctionJob() = default; @@ -324,6 +326,8 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) : m_Log(logging::Get("apply")) , m_CasStore(Store) @@ -434,6 +438,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, SharedBuffer Decompressed = DataView.Decompress(); const uint64_t DecompressedSize = DataView.GetRawSize(); + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; @@ -464,9 +470,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -483,6 +495,9 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -580,6 +595,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const IoHash DataHash = Attachment.GetHash(); CompressedBuffer DataView = Attachment.AsCompressedBinary(); + ZEN_UNUSED(DataHash); + const uint64_t CompressedSize = DataView.GetCompressedSize(); TotalAttachmentBytes += CompressedSize; @@ -605,8 +622,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; + + default: + break; } break; + + default: + break; } }, HttpVerb::kPost); @@ -844,3 +867,5 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/compute/apply.h b/zenserver/compute/apply.h index 86b262213..2506e7448 100644 --- a/zenserver/compute/apply.h +++ b/zenserver/compute/apply.h @@ -2,6 +2,14 @@ #pragma once +#include <zencore/zencore.h> + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinary.h> #include <zencore/iohash.h> #include <zencore/logging.h> @@ -50,3 +58,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/config.cpp b/zenserver/config.cpp index f512f8015..94226ef26 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -9,16 +9,16 @@ #include <zencore/string.h> #include <zenhttp/zenhttp.h> -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> -#pragma warning(pop) - #include <fmt/format.h> #include <zencore/logging.h> #include <sol/sol.hpp> +ZEN_THIRD_PARTY_INCLUDES_END -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif #if ZEN_PLATFORM_WINDOWS @@ -260,8 +260,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z if (result.count("help")) { zen::logging::ConsoleLog().info("{}", options.help()); +#if ZEN_PLATFORM_WINDOWS zen::logging::ConsoleLog().info("Press any key to exit!"); _getch(); +#else + // Assume the user's in a terminal on all other platforms and that + // they'll use less/more/etc. if need be. +#endif exit(0); } @@ -286,7 +291,7 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv using namespace fmt::literals; std::filesystem::path ConfigScript = DataRoot / "zen_cfg.lua"; - zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript.native().c_str()); + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript); if (LuaScript) { @@ -301,6 +306,7 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv // any more than it needs to lua.set_function("getenv", [&](const std::string env) -> sol::object { +#if ZEN_PLATFORM_WINDOWS std::wstring EnvVarValue; size_t RequiredSize = 0; std::wstring EnvWide = zen::Utf8ToWide(env); @@ -312,6 +318,10 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv EnvVarValue.resize(RequiredSize); _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); +#else + ZEN_UNUSED(env); + return sol::make_object(lua, sol::lua_nil); +#endif }); try diff --git a/zenserver/config.h b/zenserver/config.h index 72a4f31bb..55e846352 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -4,6 +4,7 @@ #include <filesystem> #include <string> +#include <vector> #ifndef ZEN_ENABLE_MESH # define ZEN_ENABLE_MESH 0 diff --git a/zenserver/diag/diagsvcs.h b/zenserver/diag/diagsvcs.h index 61703e393..ca8a3efda 100644 --- a/zenserver/diag/diagsvcs.h +++ b/zenserver/diag/diagsvcs.h @@ -98,6 +98,9 @@ public: { case HttpVerb::kGet: return Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, u8"OK!"sv); + + default: + break; } } diff --git a/zenserver/diag/logging.cpp b/zenserver/diag/logging.cpp index 6e2559f1f..6a5c4669e 100644 --- a/zenserver/diag/logging.cpp +++ b/zenserver/diag/logging.cpp @@ -16,6 +16,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include <spdlog/sinks/stdout_color_sinks.h> ZEN_THIRD_PARTY_INCLUDES_END +#include <zencore/filesystem.h> #include <zencore/string.h> #include <memory> @@ -168,6 +169,7 @@ private: bool EnableVTMode() { +#if ZEN_PLATFORM_WINDOWS // Set output mode to handle virtual terminal sequences HANDLE hOut = GetStdHandle(STD_OUTPUT_HANDLE); if (hOut == INVALID_HANDLE_VALUE) @@ -186,6 +188,7 @@ EnableVTMode() { return false; } +#endif return true; } @@ -230,13 +233,13 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) auto ConsoleSink = std::make_shared<spdlog::sinks::ansicolor_stdout_sink_mt>(); #if 0 - auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + auto FileSink = std::make_shared<spdlog::sinks::daily_file_sink_mt>(zen::ToUtf8(LogPath), 0, 0, /* truncate */ false, uint16_t(/* max files */ 14)); #else - auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(LogPath.c_str()), + auto FileSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::ToUtf8(LogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); @@ -264,7 +267,7 @@ InitializeLogging(const ZenServerOptions& GlobalOptions) std::filesystem::path HttpLogPath = GlobalOptions.DataDir / "logs/http.log"; - auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::WideToUtf8(HttpLogPath.c_str()), + auto HttpSink = std::make_shared<spdlog::sinks::rotating_file_sink_mt>(zen::ToUtf8(HttpLogPath), /* max size */ 128 * 1024 * 1024, /* max files */ 16, /* rotate on open */ true); diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp index 98d570cfe..4bd3ec90a 100644 --- a/zenserver/experimental/frontend.cpp +++ b/zenserver/experimental/frontend.cpp @@ -54,7 +54,7 @@ body { <pre> __________ _________ __ \____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ /_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > \/ \/ \/ \/ \/ diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index 9422dd485..95f75cea5 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -1,5 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#if ZEN_PLATFORM_WINDOWS + #include "usnjournal.h" #include <zencore/except.h> @@ -344,3 +346,5 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) } } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h index db1f59abc..2f666383b 100644 --- a/zenserver/experimental/usnjournal.h +++ b/zenserver/experimental/usnjournal.h @@ -2,6 +2,8 @@ #pragma once +#if ZEN_PLATFORM_WINDOWS + #include <zencore/windows.h> #include <zencore/zencore.h> @@ -63,3 +65,5 @@ private: }; } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 73d61c124..899080696 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -11,11 +11,14 @@ #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #define USE_ROCKSDB 0 #if USE_ROCKSDB @@ -114,7 +117,7 @@ struct ProjectStore::OplogStorage : public RefCounted #if USE_ROCKSDB { - std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); + std::string RocksdbPath = ToUtf8(m_OplogStoragePath / "ops.rdb"); ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath); @@ -184,8 +187,8 @@ struct ProjectStore::OplogStorage : public RefCounted CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); m_NextOpsOffset = - Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn); + Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); Handler(Op, LogEntry); }); @@ -275,8 +278,8 @@ private: ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath) : m_OuterProject(Project) , m_CidStore(Store) -, m_OplogId(Id) , m_BasePath(BasePath) +, m_OplogId(Id) { m_Storage = new OplogStorage(this, m_BasePath); const bool StoreExists = m_Storage->Exists(); @@ -331,7 +334,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath); FileChunk.SetContentType(ZenContentType::kBinary); return FileChunk; @@ -626,7 +629,7 @@ ProjectStore::Project::Write() CbObjectWriter Cfg; Cfg << "id" << Identifier; - Cfg << "root" << WideToUtf8(RootDir.c_str()); + Cfg << "root" << ToUtf8(RootDir); Cfg << "project" << ProjectRootDir; Cfg << "engine" << EngineRootDir; @@ -725,8 +728,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) void ProjectStore::Project::DeleteOplog(std::string_view OplogId) { - bool Exists = false; - { RwLock::ExclusiveLockScope _(m_ProjectLock); @@ -734,8 +735,6 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId) if (OplogIt != m_Oplogs.end()) { - Exists = true; - m_Oplogs.erase(OplogIt); } } @@ -776,8 +775,8 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath) : m_Log(zen::logging::Get("project")) -, m_ProjectBasePath(BasePath) , m_CidStore(Store) +, m_ProjectBasePath(BasePath) { ZEN_INFO("initializing project store at '{}'", BasePath); // m_Log.set_level(spdlog::level::debug); @@ -1492,7 +1491,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) ProjectStore::Oplog& Log = *OplogIt; CbObjectWriter Cb; - Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); + Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save()); } @@ -1531,6 +1530,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete); @@ -1628,7 +1630,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) const ProjectStore::Project& Prj = *ProjectIt; CbObjectWriter Response; - Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str()); + Response << "id" << Prj.Identifier << "root" << ToUtf8(Prj.RootDir); Response.BeginArray("oplogs"sv); Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); @@ -1654,6 +1656,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete); @@ -1680,6 +1685,7 @@ HttpProjectService::HandleRequest(HttpServerRequest& Request) ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: @@ -1712,6 +1718,12 @@ public: } } }; +#else +struct AnyUserSecurityAttributes +{ + int Attributes() { return 0666; } +}; +#endif // ZEN_PLATFORM_WINDOWS ////////////////////////////////////////////////////////////////////////// @@ -1782,6 +1794,7 @@ private: asio::thread_pool m_WorkerThreadPool; asio::io_context m_IoContext; +#if ZEN_PLATFORM_WINDOWS class PipeConnection { enum PipeState @@ -1940,6 +1953,15 @@ private: uint8_t m_MsgBuffer[16384]; }; +#else + class PipeConnection + { + public: + PipeConnection(LocalProjectImpl*) {} + void Accept() {} + void Disconnect() {} + }; +#endif AnyUserSecurityAttributes m_AnyUserSecurityAttributes; std::vector<PipeConnection*> m_ServicePipes; diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index c9f49217a..cebe26886 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -97,7 +97,6 @@ public: const std::string& OplogId() const { return m_OplogId; } - const std::wstring& TempDir() const { return m_TempPath.native(); } const std::filesystem::path& TempPath() const { return m_TempPath; } spdlog::logger& Log() { return m_OuterProject->Log(); } diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 55695ac9c..013aa28b8 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -2,6 +2,8 @@ #include "launch.h" +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> #include <zencore/filesystem.h> @@ -10,16 +12,17 @@ #include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/windows.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> +ZEN_THIRD_PARTY_INCLUDES_START #include <AccCtrl.h> #include <AclAPI.h> +#include <atlbase.h> #include <sddl.h> - #include <UserEnv.h> +ZEN_THIRD_PARTY_INCLUDES_END #pragma comment(lib, "UserEnv.lib") -#include <atlbase.h> #include <filesystem> #include <span> @@ -127,6 +130,8 @@ BasicJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +//////////////////////////////////////////////////////////////////////////////// + struct SandboxedJob { SandboxedJob() = default; @@ -317,6 +322,8 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) return true; } +//////////////////////////////////////////////////////////////////////////////// + HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) : m_Log(logging::Get("exec")) , m_CasStore(Store) @@ -336,6 +343,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -362,6 +372,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -413,6 +426,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } break; + + default: + break; } }, HttpVerb::kPost); @@ -496,6 +512,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -530,3 +549,5 @@ HttpLaunchService::CreateNewSandbox() } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/testing/launch.h b/zenserver/testing/launch.h index 49f12e2ec..13e3db4df 100644 --- a/zenserver/testing/launch.h +++ b/zenserver/testing/launch.h @@ -2,6 +2,14 @@ #pragma once +#include <zencore/zencore.h> + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES ZEN_PLATFORM_WINDOWS +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/logging.h> #include <zenhttp/httpserver.h> @@ -36,3 +44,5 @@ private: }; } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/jupiter.cpp b/zenserver/upstream/jupiter.cpp index 4caa5c8df..af416c182 100644 --- a/zenserver/upstream/jupiter.cpp +++ b/zenserver/upstream/jupiter.cpp @@ -2,7 +2,6 @@ #include "jupiter.h" -#include "cache/structuredcachestore.h" #include "diag/formatters.h" #include "diag/logging.h" diff --git a/zenserver/upstream/upstreamapply.cpp b/zenserver/upstream/upstreamapply.cpp index 3f1b0d8f9..3c67779c4 100644 --- a/zenserver/upstream/upstreamapply.cpp +++ b/zenserver/upstream/upstreamapply.cpp @@ -1,6 +1,9 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "upstreamapply.h" + +#if ZEN_WITH_COMPUTE_SERVICES + #include "jupiter.h" #include "zen.h" @@ -1490,3 +1493,5 @@ MakeHordeUpstreamEndpoint(const CloudCacheClientOptions& Options, CasStore& CasS } } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamapply.h b/zenserver/upstream/upstreamapply.h index 98f193c02..8196c3b40 100644 --- a/zenserver/upstream/upstreamapply.h +++ b/zenserver/upstream/upstreamapply.h @@ -2,6 +2,10 @@ #pragma once +#include "compute/apply.h" + +#if ZEN_WITH_COMPUTE_SERVICES + #include <zencore/compactbinarypackage.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> @@ -170,3 +174,5 @@ std::unique_ptr<UpstreamApplyEndpoint> MakeHordeUpstreamEndpoint(const CloudCach CidStore& CidStore); } // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 085932685..ade71c5d2 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -271,7 +271,7 @@ namespace detail { if (CacheRecord.Type == ZenContentType::kBinary) { CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { if (m_UseLegacyDdc) { @@ -756,7 +756,7 @@ namespace detail { Package.Save(MemStream); IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, @@ -774,7 +774,7 @@ namespace detail { for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCachePayload(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, @@ -797,7 +797,7 @@ namespace detail { } Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type); diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 5efe19094..1fbfed7dd 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -10,14 +10,14 @@ #include <zencore/uid.h> #include <zencore/zencore.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <asio.hpp> #include <chrono> +#include <list> struct ZenCacheValue; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 23cefb7b5..4af9926a6 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -2,6 +2,10 @@ #include "service.h" +#include <zencore/zencore.h> + +#if ZEN_PLATFORM_WINDOWS + #include <zencore/except.h> #include <zencore/zencore.h> @@ -638,3 +642,5 @@ SvcReportEvent(LPTSTR szFunction) // DeregisterEventSource(hEventSource); //} } + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/xmake.lua b/zenserver/xmake.lua index bba9b6ba5..b29d2cef3 100644 --- a/zenserver/xmake.lua +++ b/zenserver/xmake.lua @@ -19,6 +19,7 @@ target("zenserver") end add_options("vfs") + add_options("compute") add_packages( "vcpkg::sentry-native", @@ -29,6 +30,10 @@ target("zenserver") "vcpkg::http-parser" ) + if is_plat("linux") then + add_syslinks("dl") + end + add_packages( "vcpkg::cxxopts", "vcpkg::mimalloc") diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9a8090fc0..8b5ec79b6 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -12,13 +12,16 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenhttp/httpserver.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/cidstore.h> #include <zenutil/zenserverprocess.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #if ZEN_USE_MIMALLOC ZEN_THIRD_PARTY_INCLUDES_START # include <mimalloc-new-delete.h> @@ -138,7 +141,7 @@ namespace utils { if (!ErrorCode) { - for (const asio::ip::tcp::endpoint& Ep : Endpoints) + for (const asio::ip::tcp::endpoint Ep : Endpoints) { OutEndpoints.push_back("http://{}:{}"_format(Ep.address().to_string(), Ep.port())); } @@ -219,6 +222,7 @@ public: m_HttpProjectService.reset(new zen::HttpProjectService{*m_CidStore, m_ProjectStore}); m_LocalProjectService = zen::LocalProjectService::New(*m_CasStore, m_ProjectStore); +#if ZEN_WITH_COMPUTE_SERVICES ZEN_INFO("instantiating compute services"); std::filesystem::path SandboxDir = m_DataRoot / "exec" / "sandbox"; @@ -228,6 +232,7 @@ public: std::filesystem::path ApplySandboxDir = m_DataRoot / "exec" / "apply"; zen::CreateDirectories(ApplySandboxDir); m_HttpFunctionService = std::make_unique<zen::HttpFunctionService>(*m_CasStore, *m_CidStore, ApplySandboxDir); +#endif // ZEN_WITH_COMPUTE_SERVICES if (ServiceConfig.StructuredCacheEnabled) { @@ -265,6 +270,7 @@ public: m_Http->RegisterService(*m_StructuredCacheService); } +#if ZEN_WITH_COMPUTE_SERVICES if (m_HttpLaunchService) { m_Http->RegisterService(*m_HttpLaunchService); @@ -274,6 +280,7 @@ public: { m_Http->RegisterService(*m_HttpFunctionService); } +#endif // ZEN_WITH_COMPUTE_SERVICES m_FrontendService = std::make_unique<HttpFrontendService>(m_ContentRoot); @@ -323,7 +330,7 @@ public: if (m_DebugOptionForcedCrash) { - __debugbreak(); + ZEN_DEBUG_BREAK(); } const bool IsInteractiveMode = zen::IsInteractiveSession() && !m_TestMode; @@ -384,7 +391,7 @@ public: for (auto& PidEntry : m_ServerEntry->SponsorPids) { - if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed)) + if (uint32_t ThisPid = PidEntry.load(std::memory_order_relaxed)) { if (PidEntry.compare_exchange_strong(ThisPid, 0)) { @@ -503,13 +510,15 @@ private: zen::HttpCasService m_CasService{*m_CasStore}; zen::RefPtr<zen::ProjectStore> m_ProjectStore; zen::Ref<zen::LocalProjectService> m_LocalProjectService; - std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; std::unique_ptr<zen::HttpProjectService> m_HttpProjectService; std::unique_ptr<zen::HttpStructuredCacheService> m_StructuredCacheService; zen::HttpAdminService m_AdminService; zen::HttpHealthService m_HealthService; zen::Mesh m_ZenMesh{m_IoContext}; +#if ZEN_WITH_COMPUTE_SERVICES + std::unique_ptr<zen::HttpLaunchService> m_HttpLaunchService; std::unique_ptr<zen::HttpFunctionService> m_HttpFunctionService; +#endif std::unique_ptr<zen::HttpFrontendService> m_FrontendService; bool m_DebugOptionForcedCrash = false; @@ -733,6 +742,8 @@ ZenServer::InitializeStructuredCache(ZenServiceConfig& ServiceConfig) } // namespace zen +#if ZEN_PLATFORM_WINDOWS + class ZenWindowsService : public WindowsService { public: @@ -783,7 +794,7 @@ ZenWindowsService::Run() auto MakeLockData = [&] { CbObjectWriter Cbo; - Cbo << "pid" << _getpid() << "data" << ToUtf8(GlobalOptions.DataDir) << "port" << GlobalOptions.BasePort << "session_id" + Cbo << "pid" << zen::GetCurrentProcessId() << "data" << ToUtf8(GlobalOptions.DataDir) << "port" << GlobalOptions.BasePort << "session_id" << GetSessionId() << "ready" << IsReady; return Cbo.Save(); }; @@ -890,6 +901,10 @@ ZenWindowsService::Run() return 0; } +#endif // ZEN_PLATFORM_WINDOWS + +//////////////////////////////////////////////////////////////////////////////// + #if ZEN_WITH_TESTS int test_main(int argc, char** argv) @@ -927,6 +942,7 @@ main(int argc, char* argv[]) try { +#if ZEN_PLATFORM_WINDOWS ZenServerOptions GlobalOptions; ZenServiceConfig ServiceConfig; ParseGlobalCliOptions(argc, argv, GlobalOptions, ServiceConfig); @@ -937,7 +953,6 @@ main(int argc, char* argv[]) std::filesystem::create_directories(GlobalOptions.DataDir); } -#if ZEN_PLATFORM_WINDOWS if (GlobalOptions.InstallService) { WindowsService::Install(); @@ -951,10 +966,12 @@ main(int argc, char* argv[]) std::exit(0); } -#endif ZenWindowsService App(GlobalOptions, ServiceConfig); return App.ServiceMain(); +#else + return 1; +#endif } catch (std::exception& Ex) { diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 9ed70a5ec..938c0f9e2 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -9,6 +9,14 @@ #include <zencore/testing.h> #include <zencore/testutils.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <fcntl.h> +# include <sys/file.h> +# include <sys/stat.h> +#endif + #include <fmt/format.h> #include <gsl/gsl-lite.hpp> @@ -38,6 +46,7 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& { Ec.clear(); +#if ZEN_PLATFORM_WINDOWS const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = FILE_SHARE_READ; @@ -63,6 +72,19 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& return; } +#else + int OpenFlags = O_RDWR; + OpenFlags |= IsCreate ? O_CREAT|O_TRUNC : 0; + + int Fd = open(FileName.c_str(), OpenFlags, 0666); + if (Fd < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + return; + } + + void* FileHandle = (void*)(uintptr_t(Fd)); +#endif m_FileHandle = FileHandle; } @@ -72,7 +94,12 @@ BasicFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS ::CloseHandle(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + close(Fd); +#endif m_FileHandle = nullptr; } } @@ -86,6 +113,7 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -95,6 +123,12 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset); + bool Success = (BytesRead > 0); +#endif if (!Success) { @@ -161,6 +195,7 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -169,6 +204,12 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); + bool Success = (BytesWritten > 0); +#endif if (!Success) { @@ -210,16 +251,29 @@ BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) void BasicFile::Flush() { +#if ZEN_PLATFORM_WINDOWS FlushFileBuffers(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + fsync(Fd); +#endif } uint64_t BasicFile::FileSize() { +#if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); return uint64_t(liFileSize.QuadPart); +#else + int Fd = int(uintptr_t(m_FileHandle)); + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + return uint64_t(Stat.st_size); +#endif } ////////////////////////////////////////////////////////////////////////// @@ -234,11 +288,16 @@ TemporaryFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); +#else + std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); + unlink(FilePath.c_str()); +#endif BasicFile::Close(); } @@ -280,6 +339,7 @@ LockFile::~LockFile() void LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec) { +#if ZEN_PLATFORM_WINDOWS Ec.clear(); const DWORD dwCreationDisposition = CREATE_ALWAYS; @@ -302,6 +362,26 @@ LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_co return; } +#elif ZEN_PLATFORM_LINUX + int Fd = open(FileName.c_str(), O_RDWR|O_CREAT, 0666); + if (Fd < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + return; + } + + int LockRet = flock(Fd, LOCK_EX); + if (LockRet < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + close(Fd); + return; + } + + void* FileHandle = (void*)uintptr_t(Fd); +#else +# error check flock() support +#endif m_FileHandle = FileHandle; diff --git a/zenstore/CAS.cpp b/zenstore/cas.cpp index a4bbfa340..4268e314b 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/cas.cpp @@ -180,7 +180,7 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) id.ToString(manifest); Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); + Marker.Write(manifest.c_str(), uint32_t(manifest.Size()), 0); } } diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index 2bac6affd..ae2ea973b 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -2,7 +2,7 @@ #include <zenstore/cas.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -121,7 +121,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler) m_File.Read(ReadBuffer.data(), LogDataSize, LogBaseOffset); - for (int i = 0; i < LogEntryCount; ++i) + for (int i = 0; i < int(LogEntryCount); ++i) { Handler(ReadBuffer.data() + (i * m_RecordSize)); } @@ -135,7 +135,7 @@ CasLogFile::Append(const void* DataPointer, uint64_t DataSize) uint64_t AppendOffset = m_AppendOffset.fetch_add(DataSize); std::error_code Ec; - m_File.Write(DataPointer, gsl::narrow<DWORD>(DataSize), AppendOffset, Ec); + m_File.Write(DataPointer, gsl::narrow<uint32_t>(DataSize), AppendOffset, Ec); if (Ec) { diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 7a5d7bcf4..c67269e0d 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -5,7 +5,7 @@ #include <zencore/compress.h> #include <zencore/filesystem.h> #include <zencore/logging.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/caslog.h> #include <filesystem> diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 612f87c7c..f80bc85c4 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -2,7 +2,7 @@ #include <zenstore/cas.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/logging.h> diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index a512c3d93..a23b80828 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -9,11 +9,14 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + namespace zen { ////////////////////////////////////////////////////////////////////////// diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index a37450cd8..6d385a208 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "FileCas.h" +#include "filecas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -23,7 +23,9 @@ #include <unordered_map> ZEN_THIRD_PARTY_INCLUDES_START -#include <atlfile.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlfile.h> +#endif ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -33,7 +35,6 @@ using namespace fmt::literals; FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash) { ShardedPath.Append(RootPath.c_str()); - ShardedPath.Append(std::filesystem::path::preferred_separator); ExtendableStringBuilder<64> HashString; ChunkHash.ToHexString(HashString); @@ -53,13 +54,14 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo // would probably be a good idea to measure performance for different // policies and chunk counts + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str, str + 3); - ShardedPath.Append(std::filesystem::path::preferred_separator); + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str + 3, str + 5); Shard2len = ShardedPath.Size(); - ShardedPath.Append(std::filesystem::path::preferred_separator); + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str + 5, str + 40); } @@ -84,6 +86,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) { ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash); + RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); + +#if ZEN_PLATFORM_WINDOWS const HANDLE ChunkFileHandle = FileRef.FileHandle; auto DeletePayloadFileOnClose = [&] { @@ -104,8 +109,6 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) // // Future improvement: maintain Bloom filter to avoid expensive file system probes? - RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); - { CAtlFile PayloadFile; @@ -224,6 +227,49 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) ChunkHash); DeletePayloadFileOnClose(); +#elif ZEN_PLATFORM_LINUX + std::filesystem::path SourcePath = PathFromHandle(FileRef.FileHandle); + std::filesystem::path DestPath = Name.ShardedPath.c_str(); + int Ret = link(SourcePath.c_str(), DestPath.c_str()); + if (Ret < 0 && zen::GetLastError() == ENOENT) + { + // Destination directory doesn't exist. Create it any try again. + CreateDirectories(DestPath.parent_path().c_str()); + Ret = link(SourcePath.c_str(), DestPath.c_str()); + } + int LinkError = zen::GetLastError(); + + // Unlink the file. If the path to unlink didn't exist someone else + // beat us to it and that is hunky-dory. + if (unlink(SourcePath.c_str()) < 0) + { + int UnlinkError = zen::GetLastError(); + if (UnlinkError != ENOENT) + { + ZEN_WARN("unlink of CAS payload file failed ('{}')", GetSystemErrorAsString(UnlinkError)); + } + } + + // It is possible that someone beat us to it in linking the file. In that + // case a "file exists" error is okay. All others are not. + if (Ret < 0) + { + if (LinkError == EEXIST) + { + return CasStore::InsertResult{.New = false}; + } + + ZEN_WARN("link of CAS payload file failed ('{}'), falling back to regular write for insert of {}", + GetSystemErrorAsString(LinkError), + ChunkHash); + } + else + { + return CasStore::InsertResult{.New = true}; + } +#else +# error check link/unlink for this platform +#endif // ZEN_PLATFORM_* } return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash); @@ -238,6 +284,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize // // Future improvement: maintain Bloom filter to avoid expensive file system probes? +#if ZEN_PLATFORM_WINDOWS CAtlFile PayloadFile; HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); @@ -250,9 +297,18 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize } PayloadFile.Close(); +#elif ZEN_PLATFORM_LINUX + if (access(Name.ShardedPath.c_str(), F_OK) == 0) + { + return CasStore::InsertResult{.New = false}; + } +#else +# error Check access() for this platform +#endif RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); +#if ZEN_PLATFORM_WINDOWS // For now, use double-checked locking to see if someone else was first hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); @@ -286,6 +342,42 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize { ThrowSystemException(hRes, "Failed to open shard file '{}'"_format(WideToUtf8(Name.ShardedPath))); } +#else + // Attempt to exclusively create the file. + auto InternalCreateFile = [&] { return open(Name.ShardedPath.c_str(), O_WRONLY|O_CREAT|O_EXCL, 0666); }; + int Fd = InternalCreateFile(); + if (Fd < 0) + { + switch (zen::GetLastError()) + { + case EEXIST: + // Another thread has beat us to it so we're golden. + return {.New = false}; + + case ENOENT: + if (zen::CreateDirectories(std::string_view(Name.ShardedPath.c_str(), Name.Shard2len))) + { + Fd = InternalCreateFile(); + if (Fd >= 0) + { + break; + } + } + ThrowLastError("Failed creating shard directory '{}'"_format(Name.ShardedPath)); + + default: + ThrowLastError("Unexpected error occurred opening shard file '{}'"_format(Name.ShardedPath)); + } + } + + struct FdWrapper + { + ~FdWrapper() { Close(); } + void Write(const void* Cursor, size_t Size) { (void)!write(Fd, Cursor, Size); } + void Close() { if (Fd >= 0) { close(Fd); Fd = -1; } } + int Fd; + } PayloadFile = { Fd }; +#endif // ZEN_PLATFORM_WINDOWS size_t ChunkRemain = ChunkSize; auto ChunkCursor = reinterpret_cast<const uint8_t*>(ChunkData); @@ -337,7 +429,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) { ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash); - ZEN_DEBUG("deleting CAS payload file '{}'", WideToUtf8(Name.ShardedPath)); + ZEN_DEBUG("deleting CAS payload file '{}'", Name.ShardedPath.ToUtf8()); std::filesystem::remove(Name.ShardedPath.c_str(), Ec); } @@ -362,13 +454,13 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& struct Visitor : public FileSystemTraversal::TreeVisitor { Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {} - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override { ZEN_UNUSED(FileSize); std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); - std::wstring PathString = RelPath.native(); + std::filesystem::path::string_type PathString = RelPath.native(); if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2))) { @@ -378,12 +470,15 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& } PathString.append(File); - StringBuilder<64> Utf8; - WideToUtf8(PathString, Utf8); - // TODO: should validate that we're actually dealing with a valid hex string here +#if ZEN_PLATFORM_WINDOWS + StringBuilder<64> Utf8; + WideToUtf8(PathString, Utf8); IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()}); +#else + IoHash NameHash = IoHash::FromHexString(PathString); +#endif BasicFile PayloadFile; std::error_code Ec; @@ -397,7 +492,7 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& } virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const std::wstring_view& DirectoryName) + [[maybe_unused]] const path_view& DirectoryName) { return true; } diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 14314ce52..a86232588 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -4,9 +4,9 @@ #include <zencore/zencore.h> +#include <zencore/filesystem.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> -#include <zencore/string.h> #include <zencore/thread.h> #include <zenstore/cas.h> @@ -52,8 +52,8 @@ private: { ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash); - size_t Shard2len = 0; - ExtendableWideStringBuilder<128> ShardedPath; + size_t Shard2len = 0; + ExtendablePathBuilder<128> ShardedPath; }; }; diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index e4414787c..2df016c76 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -5,7 +5,6 @@ #include "zenstore.h" #include <zencore/iobuffer.h> -#include <zencore/windows.h> #include <filesystem> #include <functional> diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/cas.h index 86e7e78d9..7f8b5bfa2 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/cas.h @@ -41,9 +41,9 @@ public: void AddChunksToSet(std::span<const IoHash> HashesToAdd); void RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate); void IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback); - inline [[nodiscard]] bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } - inline [[nodiscard]] bool IsEmpty() const { return m_ChunkSet.empty(); } - inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); } + [[nodiscard]] inline bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } + [[nodiscard]] inline bool IsEmpty() const { return m_ChunkSet.empty(); } + [[nodiscard]] inline size_t GetSize() const { return m_ChunkSet.size(); } private: // Q: should we protect this with a lock, or is that a higher level concern? diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index 00b987383..1c6ee0bf6 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -8,10 +8,13 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <functional> namespace zen { diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 5f567e7fc..871ac553e 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -6,11 +6,9 @@ #include <tsl/robin_map.h> #include <zencore/iohash.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> -namespace std::filesystem { -class path; -} +#include <filesystem> namespace zen { diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp index d852fa64b..cc4a34fdb 100644 --- a/zenstore/zenstore.cpp +++ b/zenstore/zenstore.cpp @@ -2,7 +2,7 @@ #include "zenstore/zenstore.h" -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/basicfile.h> #include "filecas.h" diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj index eb2ecd02b..9431b7b11 100644 --- a/zenstore/zenstore.vcxproj +++ b/zenstore/zenstore.vcxproj @@ -28,7 +28,7 @@ <ClInclude Include="include\zenstore\cidstore.h" /> <ClInclude Include="include\zenstore\gc.h" /> <ClInclude Include="include\zenstore\scrub.h" /> - <ClInclude Include="include\zenstore\CAS.h" /> + <ClInclude Include="include\zenstore\cas.h" /> <ClInclude Include="include\zenstore\caslog.h" /> <ClInclude Include="include\zenstore\zenstore.h" /> </ItemGroup> diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters index 8a52c69f6..904de0748 100644 --- a/zenstore/zenstore.vcxproj.filters +++ b/zenstore/zenstore.vcxproj.filters @@ -14,7 +14,7 @@ <ItemGroup> <ClInclude Include="compactcas.h" /> <ClInclude Include="filecas.h" /> - <ClInclude Include="include\zenstore\CAS.h" /> + <ClInclude Include="include\zenstore\cas.h" /> <ClInclude Include="include\zenstore\caslog.h" /> <ClInclude Include="include\zenstore\gc.h" /> <ClInclude Include="include\zenstore\scrub.h" /> diff --git a/zenutil/include/zenutil/zenserverprocess.h b/zenutil/include/zenutil/zenserverprocess.h index 8a4f9604d..55b9a50cd 100644 --- a/zenutil/include/zenutil/zenserverprocess.h +++ b/zenutil/include/zenutil/zenserverprocess.h @@ -67,8 +67,8 @@ struct ZenServerInstance private: ZenServerEnvironment& m_Env; ProcessHandle m_Process; - Event m_ReadyEvent; - Event m_ShutdownEvent; + NamedEvent m_ReadyEvent; + NamedEvent m_ShutdownEvent; bool m_Terminate = false; std::filesystem::path m_TestDir; bool m_MeshEnabled = false; diff --git a/zenutil/xmake.lua b/zenutil/xmake.lua index 7cd29a86f..b7f08afb9 100644 --- a/zenutil/xmake.lua +++ b/zenutil/xmake.lua @@ -3,4 +3,8 @@ target('zenutil') add_files("**.cpp") add_includedirs("include", {public=true}) add_deps("zencore") - add_packages("vcpkg::spdlog")
\ No newline at end of file + add_packages("vcpkg::spdlog") + + if is_plat("linux") then + add_syslinks("rt") + end
\ No newline at end of file diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index 4098954a8..8dd93b32e 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -8,18 +8,25 @@ #include <zencore/logging.h> #include <zencore/session.h> #include <zencore/string.h> +#include <zencore/thread.h> -#include <atlbase.h> -#include <shellapi.h> +#include <atomic> #include <source_location> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlbase.h> +# include <shellapi.h> +# include <zencore/windows.h> +#else +# include <sys/mman.h> +#endif ////////////////////////////////////////////////////////////////////////// namespace zen { namespace zenutil { +#if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: @@ -53,6 +60,7 @@ namespace zenutil { } } }; +#endif // ZEN_PLATFORM_WINDOWS } // namespace zenutil @@ -72,59 +80,85 @@ ZenServerState::~ZenServerState() m_OurEntry = nullptr; } +#if ZEN_PLATFORM_WINDOWS if (m_Data) { UnmapViewOfFile(m_Data); - m_Data = nullptr; } if (m_hMapFile) { CloseHandle(m_hMapFile); } +#else + if (m_Data != nullptr) + { + munmap(m_Data, m_MaxEntryCount * sizeof(ZenServerEntry)); + } + + int Fd = int(intptr_t(m_hMapFile)); + close(Fd); +#endif + + m_Data = nullptr; } void ZenServerState::Initialize() { + size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); + +#if ZEN_PLATFORM_WINDOWS // TODO: there's a small chance of a race here, this logic could be tightened up with a mutex to // ensure only a single process at a time creates the mapping - if (HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap")) - { - m_hMapFile = hMap; - } - else + HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); + if (hMap == NULL) { // Security attributes to enable any user to access state zenutil::AnyUserSecurityAttributes Attrs; - hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file - Attrs.Attributes(), // allow anyone to access - PAGE_READWRITE, // read/write access - 0, // maximum object size (high-order DWORD) - m_MaxEntryCount * sizeof(ZenServerEntry), // maximum object size (low-order DWORD) - L"Global\\ZenMap"); // name of mapping object + hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file + Attrs.Attributes(), // allow anyone to access + PAGE_READWRITE, // read/write access + 0, // maximum object size (high-order DWORD) + DWORD(MapSize), // maximum object size (low-order DWORD) + L"Global\\ZenMap"); // name of mapping object if (hMap == NULL) { ThrowLastError("Could not open or create file mapping object for Zen server state"); } - - m_hMapFile = hMap; } - void* pBuf = MapViewOfFile(m_hMapFile, // handle to map object + void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_ALL_ACCESS, // read/write permission 0, // offset high 0, // offset low - m_MaxEntryCount * sizeof(ZenServerEntry)); + DWORD(MapSize)); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } +#else + int Fd = shm_open("UnrealEngineZen", O_RDWR|O_CREAT, 0666); + if (Fd < 0) + { + ThrowLastError("Could not open a shared memory object"); + } + void* hMap = (void*)intptr_t(Fd); + + ftruncate(Fd, MapSize); + + void* pBuf = mmap(nullptr, MapSize, PROT_READ|PROT_WRITE, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + ThrowLastError("Could not map view of Zen server state"); + } +#endif + m_hMapFile = hMap; m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); m_IsReadOnly = false; } @@ -132,27 +166,42 @@ ZenServerState::Initialize() bool ZenServerState::InitializeReadOnly() { - if (HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap")) - { - m_hMapFile = hMap; - } - else + size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); + +#if ZEN_PLATFORM_WINDOWS + HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); + if (hMap == NULL) { return false; } - void* pBuf = MapViewOfFile(m_hMapFile, // handle to map object + void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_READ, // read permission 0, // offset high 0, // offset low - m_MaxEntryCount * sizeof(ZenServerEntry)); + MapSize); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } +#else + int Fd = shm_open("UnrealEngineZen", O_RDONLY, 0666); + if (Fd < 0) + { + return false; + } + void* hMap = (void*)intptr_t(Fd); + + void* pBuf = mmap(nullptr, MapSize, PROT_READ, MAP_PRIVATE, Fd, 0); + if (pBuf == MAP_FAILED) + { + ThrowLastError("Could not map read-only view of Zen server state"); + } +#endif - m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); + m_hMapFile = hMap; + m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); return true; } @@ -187,7 +236,7 @@ ZenServerState::Register(int ListenPort) { ZenServerEntry& Entry = m_Data[i]; - if (Entry.ListenPort.load(std::memory_order::memory_order_relaxed) == 0) + if (Entry.ListenPort.load(std::memory_order_relaxed) == 0) { uint16_t Expected = 0; if (Entry.ListenPort.compare_exchange_strong(Expected, uint16_t(ListenPort))) @@ -280,7 +329,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd) { for (std::atomic<uint32_t>& PidEntry : SponsorPids) { - if (PidEntry.load(std::memory_order::memory_order_relaxed) == 0) + if (PidEntry.load(std::memory_order_relaxed) == 0) { uint32_t Expected = 0; if (PidEntry.compare_exchange_strong(Expected, uint16_t(PidToAdd))) @@ -289,7 +338,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd) return true; } } - else if (PidEntry.load(std::memory_order::memory_order_relaxed) == PidToAdd) + else if (PidEntry.load(std::memory_order_relaxed) == PidToAdd) { // Success, the because pid is already in the list return true; @@ -404,12 +453,13 @@ ZenServerInstance::Shutdown() void ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs) { +#if ZEN_PLATFORM_WINDOWS ZEN_ASSERT(!m_Process.IsValid()); // Only spawn once const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = BaseDir / "zenserver.exe"; - const int MyPid = _getpid(); + const int MyPid = zen::GetCurrentProcessId(); const int ChildId = ++ChildIdCounter; ExtendableStringBuilder<32> ChildEventName; @@ -555,6 +605,9 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr } m_ReadyEvent = std::move(ChildEvent); +#else + ZEN_UNUSED(BasePort, AdditionalServerArgs); +#endif // ZEN_PLATFORM_WINDOWS } void |