diff options
74 files changed, 1100 insertions, 337 deletions
diff --git a/.gitignore b/.gitignore index bf237a7b3..103d3bd3f 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ +vsxmake20*/ +vs20*/ # Visual Studio Code options directory .vscode/ @@ -217,6 +219,7 @@ __pycache__/ .minio_data/ .test/ .xmake/ +.gdb_history # Tags TAGS @@ -55,11 +55,67 @@ You can then build from the command line: It's also possible to generate project files for Visual Studio via -`d:\zen> xmake project -k vsxmake` +`d:\zen> xmake project -k vsxmake -a x64` ## Building on Linux -... coming soon +The following instructions have been collated using Ubuntu 20.04. + +At the time of writing only GCC v11 supports the C++20 features used by Zen. As +this version is not available in the default package repositories we need to +get GCC from one of Ubuntu's toolchain repositories; + +``` +sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test +sudo apt install -y gcc-11 +gcc-11 --version +``` + +Next we need the `xmake` build system. For this we will download and install +`xmake` as a `.deb` package. Check the project's release page for more up to +date `.deb` files; + +``` +wget https://github.com/xmake-io/xmake/releases/download/v2.5.7/xmake-v2.5.7.amd64.deb +sudo dpkg -i xmake-v2.5.7.amd64.deb +xmake --version +``` + +For some of Zen's third party dependencies are provided by Microsoft's `vcpkg` +C++ library manager. After cloning the project there is a initialisation step; + +``` +git clone https://github.com/microsoft/vcpkg.git ~/zen/vcpkg +~/zen/vcpkg/bootstrap-vcpkg.sh +``` + +`xmake` uses an environment variable to find `vcpkg`. Alternatively this can be +done by including `VCPKG_ROOT=...` on the command line when invoking `xmake`; + +``` +export VCPKG_ROOT=~/zen/vcpkg +``` + +Clone the Zen project and tell `xmake` to use the correct GCC version; + +``` +git clone https://github.com/EpicGames/zen.git ~/zen/main +cd ~/zen/main +xmake config --plat=linux --cxx=g++-11 --cc=gcc-11 [--mode=debug] +``` + +The `--mode=debug` is optionally required to change the build variant. This +cannot be done when executing the build and the setting of configuration options +can not be composed across multiple `xmake config` calls. + +Now we are ready to build Zen. The `-y` skips `xmake` from prompting about +updating `vcpkg` packages; + +``` +xmake build -y +``` + +The `xmake` flags `-vD` can be useful to diagnose `xmake` issues. ## Building on Mac @@ -39,27 +39,43 @@ else end if is_os("windows") then - add_defines("_CRT_SECURE_NO_WARNINGS", "_UNICODE", "UNICODE", "_WIN32_WINNT=0x0A00") + add_defines( + "_CRT_SECURE_NO_WARNINGS", + "_UNICODE", + "UNICODE", + "_CONSOLE", + "NOMINMAX", -- stop Windows SDK defining 'min' and 'max' + "NOGDI", -- otherwise Windows.h defines 'GetObject' + "WIN32_LEAN_AND_MEAN", -- cut down Windows.h + "_WIN32_WINNT=0x0A00" + ) -- add_ldflags("/MAP") end +if is_os("linux") then + add_cxxflags("-Wno-unused-value") + add_cxxflags("-Wno-strict-aliasing") + add_cxxflags("-Wno-implicit-fallthrough") + add_cxxflags("-Wno-missing-field-initializers") +end + add_defines("USE_SENTRY=1") add_defines("ZEN_USE_MIMALLOC=1") -option("vfs") - set_showmenu(true) - set_description("Enable VFS functionality") - add_defines("ZEN_WITH_VFS") -option_end() - -option("httpsys") - set_default(true) - set_showmenu(true) - set_description("Enable http.sys server") - add_defines("ZEN_WITH_HTTPSYS") -option_end() +if is_os("windows") then + option("vfs") + set_showmenu(true) + set_description("Enable VFS functionality") + add_defines("ZEN_WITH_VFS") + option_end() -add_defines("UNICODE", "_CONSOLE") + option("httpsys") + set_default(true) + set_showmenu(true) + set_description("Enable http.sys server") + add_defines("ZEN_WITH_HTTPSYS") + option_end() +end set_warnings("allextra", "error") set_languages("cxx20") diff --git a/zen/cmds/run.cpp b/zen/cmds/run.cpp index 8cbf13566..085e74061 100644 --- a/zen/cmds/run.cpp +++ b/zen/cmds/run.cpp @@ -152,7 +152,7 @@ RunCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (auto It = Visit.m_HashToFile.find(NeedHash); It != Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url("http://localhost:13337/cas"), cpr::Body((const char*)FileData.Data(), FileData.Size())); diff --git a/zen/zen.cpp b/zen/zen.cpp index 3c33ff5e0..1fe23d32b 100644 --- a/zen/zen.cpp +++ b/zen/zen.cpp @@ -264,7 +264,7 @@ main(int argc, char** argv) for (const CommandInfo& CmdInfo : Commands) { - if (_stricmp(SubCommand.c_str(), CmdInfo.CmdName) == 0) + if (StrCaseCompare(SubCommand.c_str(), CmdInfo.CmdName) == 0) { cxxopts::Options* VerbOptions = CmdInfo.Cmd->Options(); @@ -2,10 +2,9 @@ #pragma once -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <zencore/refcount.h> #include <zencore/windows.h> diff --git a/zencore/blake3.cpp b/zencore/blake3.cpp index 663f21b6d..5869f19a3 100644 --- a/zencore/blake3.cpp +++ b/zencore/blake3.cpp @@ -8,7 +8,9 @@ #include <zencore/zencore.h> #include "../3rdparty/BLAKE3/c/blake3.h" -#pragma comment(lib, "blake3.lib") +#if ZEN_PLATFORM_WINDOWS +# pragma comment(lib, "blake3.lib") +#endif #include <string.h> diff --git a/zencore/compactbinary.cpp b/zencore/compactbinary.cpp index aafb365f3..d7c4c32d0 100644 --- a/zencore/compactbinary.cpp +++ b/zencore/compactbinary.cpp @@ -18,6 +18,8 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> +#else +# include <time.h> #endif #if ZEN_WITH_TESTS @@ -46,15 +48,31 @@ IsLeapYear(int Year) return false; } +static constexpr uint64_t +GetPlatformToDateTimeBiasInSeconds() +{ +#if ZEN_PLATFORM_WINDOWS + const uint64_t PlatformEpochYear = 1601; +#else + const uint64_t PlatformEpochYear = 1970; +#endif + const uint64_t DateTimeEpochYear = 1; + return uint64_t(double(PlatformEpochYear - DateTimeEpochYear) * 365.2425) * 86400; +} + DateTime DateTime::Now() { + static const uint64_t EpochBias = GetPlatformToDateTimeBiasInSeconds(); + static const uint64_t SecsTo100nsTicks = int64_t(10e9 / 100); + #if ZEN_PLATFORM_WINDOWS FILETIME SysTime; GetSystemTimeAsFileTime(&SysTime); - return DateTime{504911232000000000ull + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime}; + return DateTime{(EpochBias * SecsTo100nsTicks) + (uint64_t(SysTime.dwHighDateTime) << 32) | SysTime.dwLowDateTime}; #else -# error Needs implementation + int64_t SecondsSinceUnixEpoch = time(nullptr); + return DateTime{(EpochBias + SecondsSinceUnixEpoch) * SecsTo100nsTicks}; #endif } @@ -82,88 +100,6 @@ DateTime::Set(int Year, int Month, int Day, int Hour, int Minute, int Second, in Second * TimeSpan::TicksPerSecond + MilliSecond * TimeSpan::TicksPerMillisecond; } -void -TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) -{ - int64_t TotalTicks = 0; - - TotalTicks += Days * TicksPerDay; - TotalTicks += Hours * TicksPerHour; - TotalTicks += Minutes * TicksPerMinute; - TotalTicks += Seconds * TicksPerSecond; - TotalTicks += FractionNano / NanosecondsPerTick; - - Ticks = TotalTicks; -} - -std::string -TimeSpan::ToString(const char* Format) const -{ - using namespace fmt::literals; - - StringBuilder<128> Result; - - Result.Append((Ticks < 0) ? '-' : '+'); - - while (*Format != '\0') - { - if ((*Format == '%') && (*++Format != '\0')) - { - switch (*Format) - { - case 'd': - Result.Append("{}"_format(GetDays())); - break; - case 'D': - Result.Append("{:08}"_format(GetDays())); - break; - case 'h': - Result.Append("{:02}"_format(GetHours())); - break; - case 'm': - Result.Append("{:02}"_format(GetMinutes())); - break; - case 's': - Result.Append("{:02}"_format(GetSeconds())); - break; - case 'f': - Result.Append("{:03}"_format(GetFractionMilli())); - break; - case 'u': - Result.Append("{:06}"_format(GetFractionMicro())); - break; - case 't': - Result.Append("{:07}"_format(GetFractionTicks())); - break; - case 'n': - Result.Append("{:09}"_format(GetFractionNano())); - break; - default: - Result.Append(*Format); - } - } - else - { - Result.Append(*Format); - } - - ++Format; - } - - return Result.ToString(); -} - -std::string -TimeSpan::ToString() const -{ - if (GetDays() == 0) - { - return ToString("%h:%m:%s.%f"); - } - - return ToString("%d.%h:%m:%s.%f"); -} - int DateTime::GetYear() const { @@ -329,6 +265,88 @@ DateTime::ToIso8601() const return ToString("%Y-%m-%dT%H:%M:%S.%sZ"); } +void +TimeSpan::Set(int Days, int Hours, int Minutes, int Seconds, int FractionNano) +{ + int64_t TotalTicks = 0; + + TotalTicks += Days * TicksPerDay; + TotalTicks += Hours * TicksPerHour; + TotalTicks += Minutes * TicksPerMinute; + TotalTicks += Seconds * TicksPerSecond; + TotalTicks += FractionNano / NanosecondsPerTick; + + Ticks = TotalTicks; +} + +std::string +TimeSpan::ToString(const char* Format) const +{ + using namespace fmt::literals; + + StringBuilder<128> Result; + + Result.Append((int64_t(Ticks) < 0) ? '-' : '+'); + + while (*Format != '\0') + { + if ((*Format == '%') && (*++Format != '\0')) + { + switch (*Format) + { + case 'd': + Result.Append("{}"_format(GetDays())); + break; + case 'D': + Result.Append("{:08}"_format(GetDays())); + break; + case 'h': + Result.Append("{:02}"_format(GetHours())); + break; + case 'm': + Result.Append("{:02}"_format(GetMinutes())); + break; + case 's': + Result.Append("{:02}"_format(GetSeconds())); + break; + case 'f': + Result.Append("{:03}"_format(GetFractionMilli())); + break; + case 'u': + Result.Append("{:06}"_format(GetFractionMicro())); + break; + case 't': + Result.Append("{:07}"_format(GetFractionTicks())); + break; + case 'n': + Result.Append("{:09}"_format(GetFractionNano())); + break; + default: + Result.Append(*Format); + } + } + else + { + Result.Append(*Format); + } + + ++Format; + } + + return Result.ToString(); +} + +std::string +TimeSpan::ToString() const +{ + if (GetDays() == 0) + { + return ToString("%h:%m:%s.%f"); + } + + return ToString("%d.%h:%m:%s.%f"); +} + StringBuilderBase& Guid::ToString(StringBuilderBase& Sb) const { @@ -1854,11 +1872,11 @@ TEST_CASE("uson.json") << "ValueTwo"; CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const std::string ValueOne = Json["KeyOne"].string_value(); const std::string ValueTwo = Json["KeyTwo"].string_value(); @@ -1879,11 +1897,11 @@ TEST_CASE("uson.json") CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const float FloatValue = float(Json["Float"].number_value()); const double DoubleValue = Json["Double"].number_value(); @@ -1904,11 +1922,11 @@ TEST_CASE("uson.json") CbObject Obj = Writer.Save(); - StringBuilder<128> Sb; - const std::string_view JsonText = Obj.ToJson(Sb).ToView(); + StringBuilder<128> Sb; + const char* JsonText = Obj.ToJson(Sb).Data(); std::string JsonError; - json11::Json Json = json11::Json::parse(JsonText.data(), JsonError); + json11::Json Json = json11::Json::parse(JsonText, JsonError); const double FloatValue = Json["FloatNan"].number_value(); const double DoubleValue = Json["DoubleNan"].number_value(); diff --git a/zencore/compactbinarybuilder.cpp b/zencore/compactbinarybuilder.cpp index fa5b6a69b..3be24c149 100644 --- a/zencore/compactbinarybuilder.cpp +++ b/zencore/compactbinarybuilder.cpp @@ -985,7 +985,12 @@ TEST_CASE("usonbuilder.string") SUBCASE("Non-ASCII String") { +#if ZEN_SIZEOF_WCHAR_T == 2 wchar_t Value[2] = {0xd83d, 0xde00}; +#else + wchar_t Value[1] = {0x1f600}; +#endif + Writer.AddString("\xf0\x9f\x98\x80"sv); Writer.AddString(std::wstring_view(Value, ZEN_ARRAY_COUNT(Value))); CbFieldIterator Fields = Writer.Save(); diff --git a/zencore/compress.cpp b/zencore/compress.cpp index 4a8d116fa..dd6484a3c 100644 --- a/zencore/compress.cpp +++ b/zencore/compress.cpp @@ -1198,7 +1198,6 @@ TEST_CASE("CompressedBuffer") SUBCASE("copy uncompressed range") { - const uint64_t BlockSize = 64 * sizeof(uint64_t); const uint64_t N = 1000; std::vector<uint64_t> ExpectedValues = GenerateData(N); diff --git a/zencore/filesystem.cpp b/zencore/filesystem.cpp index a642e2cf6..3c5ab2026 100644 --- a/zencore/filesystem.cpp +++ b/zencore/filesystem.cpp @@ -174,7 +174,8 @@ CleanDirectory(const wchar_t* DirPath) bool CreateDirectories(const std::filesystem::path& Dir) { - return std::filesystem::create_directories(Dir); + std::error_code ErrorCode; + return std::filesystem::create_directories(Dir, ErrorCode); } bool @@ -244,6 +245,7 @@ SupportsBlockRefCounting(std::filesystem::path Path) return true; #else + ZEN_UNUSED(Path); return false; #endif // ZEN_PLATFORM_WINDOWS } @@ -406,16 +408,47 @@ CloneFile(std::filesystem::path FromPath, std::filesystem::path ToPath) const bool AllOk = (TRUE == SetFileInformationByHandle(TargetFile, FileDispositionInfo, &FileDisposition, sizeof FileDisposition)); return AllOk; -#else +#elif ZEN_PLATFORM_LINUX +# if 0 + struct ScopedFd + { + ~ScopedFd() { close(Fd); } + int Fd; + }; + + // The 'from' file + int FromFd = open(FromPath.c_str(), O_RDONLY); + if (FromFd < 0) + { + return false; + } + ScopedFd $From = { FromFd }; + + // The 'to' file + int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL, 0666); + if (ToFd < 0) + { + return false; + } + ScopedFd $To = { FromFd }; + + ioctl(ToFd, FICLONE, FromFd); + + return false; +# endif // 0 + ZEN_UNUSED(FromPath, ToPath); ZEN_ERROR("CloneFile() is not implemented on this platform"); return false; +#elif ZEN_PLATFORM_MAC + /* clonefile() syscall if APFS */ +# error not implemented + return false; #endif // ZEN_PLATFORM_WINDOWS } bool CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const CopyFileOptions& Options) { -#if ZEN_PLATFORM_WINDOWS bool Success = false; if (Options.EnableClone) @@ -433,6 +466,7 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop return false; } +#if ZEN_PLATFORM_WINDOWS BOOL CancelFlag = FALSE; Success = !!::CopyFileExW(FromPath.c_str(), ToPath.c_str(), @@ -440,17 +474,54 @@ CopyFile(std::filesystem::path FromPath, std::filesystem::path ToPath, const Cop /* lpData */ nullptr, &CancelFlag, /* dwCopyFlags */ 0); +#else + using namespace fmt::literals; - if (!Success) + struct ScopedFd { ~ScopedFd() { close(Fd); } int Fd; }; + + // From file + int FromFd = open(FromPath.c_str(), O_RDONLY); + if (FromFd < 0) { - throw std::system_error(std::error_code(::GetLastError(), std::system_category()), "file copy failed"); + ThrowLastError("failed to open file {}"_format(FromPath)); } + ScopedFd $From = { FromFd }; - return Success; -#else - ZEN_ERROR("CopyFile() is not implemented on this platform"); - return false; + // To file + int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL); + if (ToFd < 0) + { + ThrowLastError("failed to create file {}"_format(ToPath)); + } + ScopedFd $To = { ToFd }; + + // Copy impl + static const size_t BufferSize = 64 << 10; + void* Buffer = malloc(BufferSize); + while (true) + { + int BytesRead = read(FromFd, Buffer, BufferSize); + if (BytesRead <= 0) + { + Success = (BytesRead == 0); + break; + } + + if (write(ToFd, Buffer, BytesRead) != BufferSize) + { + Success = false; + break; + } + } + free(Buffer); #endif // ZEN_PLATFORM_WINDOWS + + if (!Success) + { + ThrowLastError("file copy failed"sv); + } + + return true; } void @@ -505,7 +576,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, "File write failed for '{}'"_format(Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != WriteSize) + if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) { ThrowLastError("File write failed for '{}'"_format(Path)); } @@ -572,7 +643,9 @@ ReadFile(std::filesystem::path Path) int Fd = open(Path.c_str(), O_RDONLY); if (Fd < 0) { - return FileContents{.ErrorCode = std::error_code(zen::GetLastError(), std::system_category())}; + FileContents Ret; + Ret.ErrorCode = std::error_code(zen::GetLastError(), std::system_category()); + return Ret; } static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); @@ -616,12 +689,43 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi ProcessFunc(ReadBuffer.data(), dwBytesRead); } - - return true; #else - ZEN_ERROR("ScanFile() is not implemented on this platform"); - return false; + int Fd = open(Path.c_str(), O_RDONLY); + if (Fd < 0) + { + return false; + } + + bool Success = true; + + void* Buffer = malloc(ChunkSize); + while (true) + { + int BytesRead = read(Fd, Buffer, ChunkSize); + if (BytesRead < 0) + { + Success = false; + break; + } + + if (BytesRead == 0) + { + break; + } + + ProcessFunc(Buffer, BytesRead); + } + + free(Buffer); + close(Fd); + + if (!Success) + { + ThrowLastError("file scan failed"); + } #endif // ZEN_PLATFORM_WINDOWS + + return true; } std::string @@ -731,7 +835,7 @@ FileSystemTraversal::TraverseFileSystem(const std::filesystem::path& RootDir, Tr ThrowLastError("Failed to open directory for traversal: {}"_format(RootDir.c_str())); } - for (struct dirent* Entry; Entry = readdir(Dir);) + for (struct dirent* Entry; (Entry = readdir(Dir));) { const char* FileName = Entry->d_name; @@ -787,14 +891,15 @@ PathFromHandle(void* NativeHandle) return FullPath; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/fd/%d", getpid(), int(uintptr_t(NativeHandle))); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead <= 0) return std::filesystem::path(); - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS @@ -809,14 +914,15 @@ GetRunningExecutablePath() return {std::wstring_view(ExePath, PathLength)}; #elif ZEN_PLATFORM_LINUX - char Buffer[256]; - sprintf(Buffer, "/proc/%d/exe", getpid()); - ssize_t BytesRead = readlink(Buffer, Buffer, sizeof(Buffer) - 1); + char Link[256]; + char Path[64]; + sprintf(Path, "/proc/%d/exe", getpid()); + ssize_t BytesRead = readlink(Path, Link, sizeof(Link) - 1); if (BytesRead < 0) return {}; - Buffer[BytesRead] = '\0'; - return Buffer; + Link[BytesRead] = '\0'; + return Link; #else # error Unimplemented platform #endif // ZEN_PLATFORM_WINDOWS @@ -881,6 +987,50 @@ TEST_CASE("filesystem") FileSystemTraversal().TraverseFileSystem(BinPath.parent_path().parent_path(), Visitor); CHECK(Visitor.bFoundExpected); + + // Scan/read file + FileContents BinRead = ReadFile(BinPath); + std::vector<uint8_t> BinScan; + ScanFile(BinPath, 16 << 10, [&] (const void* Data, size_t Size) { + const auto* Ptr = (uint8_t*)Data; + BinScan.insert(BinScan.end(), Ptr, Ptr + Size); + }); + CHECK_EQ(BinRead.Data.size(), 1); + CHECK_EQ(BinScan.size(), BinRead.Data[0].GetSize()); +} + +TEST_CASE("PathBuilder") +{ +#if ZEN_PLATFORM_WINDOWS + const char* foo_bar = "/foo\\bar"; +#else + const char* foo_bar = "/foo/bar"; +#endif + + ExtendablePathBuilder<32> Path; + for (const char* Prefix : { "/foo", "/foo/" }) + { + Path.Reset(); + Path.Append(Prefix); + Path /= "bar"; + CHECK(Path.ToPath() == foo_bar); + } + + using fspath = std::filesystem::path; + + Path.Reset(); + Path.Append(fspath("/foo/")); + Path /= (fspath("bar")); + CHECK(Path.ToPath() == foo_bar); + +#if ZEN_PLATFORM_WINDOWS + Path.Reset(); + Path.Append(fspath(L"/\u0119oo/")); + Path /= L"bar"; + printf("%ls\n", Path.ToPath().c_str()); + CHECK(Path.ToView() == L"/\u0119oo/bar"); + CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); +# endif } #endif diff --git a/zencore/include/zencore/blockingqueue.h b/zencore/include/zencore/blockingqueue.h index 277095689..f92df5a54 100644 --- a/zencore/include/zencore/blockingqueue.h +++ b/zencore/include/zencore/blockingqueue.h @@ -3,6 +3,7 @@ #pragma once #include <atomic> +#include <condition_variable> #include <deque> #include <mutex> diff --git a/zencore/include/zencore/compactbinary.h b/zencore/include/zencore/compactbinary.h index 06331c510..0be23ea11 100644 --- a/zencore/include/zencore/compactbinary.h +++ b/zencore/include/zencore/compactbinary.h @@ -63,7 +63,7 @@ public: private: void Set(int Year, int Month, int Day, int Hours, int Minutes, int Seconds, int MilliSecond); - uint64_t Ticks; + uint64_t Ticks; // 1 tick == 0.1us == 100ns, epoch == Jan 1st 0001 }; class TimeSpan diff --git a/zencore/include/zencore/filesystem.h b/zencore/include/zencore/filesystem.h index c7ac7140d..e0fd891f1 100644 --- a/zencore/include/zencore/filesystem.h +++ b/zencore/include/zencore/filesystem.h @@ -5,6 +5,7 @@ #include "zencore.h" #include <zencore/iobuffer.h> +#include <zencore/string.h> #include <filesystem> #include <functional> @@ -58,6 +59,54 @@ ZENCORE_API bool SupportsBlockRefCounting(std::filesystem::path Path); ZENCORE_API std::string ToUtf8(const std::filesystem::path& Path); /** + * Helper class for building paths. Backed by an extendable string builder. + * + */ +#if ZEN_PLATFORM_WINDOWS + template <size_t N> + using ExtendablePathBuilderBase = ExtendableWideStringBuilder<N>; +#else + template <size_t N> + using ExtendablePathBuilderBase = ExtendableStringBuilder<N>; +#endif + +template <size_t N> +class ExtendablePathBuilder + : public ExtendablePathBuilderBase<N> +{ + using Super = ExtendablePathBuilderBase<N>; + +public: + void Append(const std::filesystem::path& Rhs) { Super::Append(Rhs.c_str()); } + void operator /= (const std::filesystem::path& Rhs) { this->operator /= (Rhs.c_str()); }; + void operator /= (const char* Rhs) { AppendSeparator(); *this << (Rhs); } + void operator /= (const wchar_t* Rhs) { AppendSeparator(); Super::Append(Rhs); } + std::filesystem::path ToPath() const { return std::filesystem::path(Super::ToView()); } + + std::string ToUtf8() const + { +#if ZEN_PLATFORM_WINDOWS + return WideToUtf8(Super::ToView()); +#else + return std::string(Super::ToView()); +#endif + } + + void AppendSeparator() + { + if ( + Super::ToView().ends_with(std::filesystem::path::preferred_separator) +#if ZEN_PLATFORM_WINDOWS + || Super::ToView().ends_with('/') +#endif + ) + return; + + Super::Append(std::filesystem::path::preferred_separator); + } +}; + +/** * Efficient file system traversal * * Uses the best available mechanism for the platform in question and could take diff --git a/zencore/include/zencore/intmath.h b/zencore/include/zencore/intmath.h index 7619e1950..0d0ceff16 100644 --- a/zencore/include/zencore/intmath.h +++ b/zencore/include/zencore/intmath.h @@ -107,7 +107,7 @@ FloorLog2(uint32_t Value) static inline uint32_t CountLeadingZeros(uint32_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; _BitScanReverse64(&Log2, (uint64_t(Value) << 1) | 1); return 32 - Log2; } @@ -115,7 +115,7 @@ CountLeadingZeros(uint32_t Value) static inline uint64_t FloorLog2_64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return Log2 & Mask; } @@ -123,7 +123,7 @@ FloorLog2_64(uint64_t Value) static inline uint64_t CountLeadingZeros64(uint64_t Value) { - unsigned long Log2; + unsigned long Log2 = 0; long Mask = -long(_BitScanReverse64(&Log2, Value) != 0); return ((63 - Log2) & Mask) | (64 & ~Mask); } diff --git a/zencore/include/zencore/iobuffer.h b/zencore/include/zencore/iobuffer.h index 88a72cbba..b1d13c58f 100644 --- a/zencore/include/zencore/iobuffer.h +++ b/zencore/include/zencore/iobuffer.h @@ -374,10 +374,8 @@ private: class IoBufferBuilder { - using path_char_t = std::filesystem::path::value_type; - public: - ZENCORE_API static IoBuffer MakeFromFile(const path_char_t* FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); + ZENCORE_API static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); ZENCORE_API static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); ZENCORE_API static IoBuffer ReadFromFileMaybe(IoBuffer& InBuffer); diff --git a/zencore/include/zencore/refcount.h b/zencore/include/zencore/refcount.h index 7167ab3b5..92ebebcfe 100644 --- a/zencore/include/zencore/refcount.h +++ b/zencore/include/zencore/refcount.h @@ -159,7 +159,7 @@ public: private: T* m_Ref = nullptr; - template<class T> + template<class U> friend class Ref; }; diff --git a/zencore/include/zencore/string.h b/zencore/include/zencore/string.h index a5ab25068..7af27ca20 100644 --- a/zencore/include/zencore/string.h +++ b/zencore/include/zencore/string.h @@ -424,7 +424,7 @@ public: inline std::wstring_view ToView() const { return std::wstring_view{Data(), Size()}; } inline std::wstring toString() const { return std::wstring{Data(), Size()}; } - inline StringBuilderImpl& operator<<(const std::u16string_view str) { return Append((const wchar_t*)str.data(), str.size()); } + inline StringBuilderImpl& operator<<(const std::wstring_view str) { return Append((const wchar_t*)str.data(), str.size()); } inline StringBuilderImpl& operator<<(const wchar_t* str) { return Append(str); } using StringBuilderImpl:: operator<<; }; @@ -460,7 +460,6 @@ std::wstring Utf8ToWide(const std::string_view& wstr); void WideToUtf8(const wchar_t* wstr, StringBuilderBase& out); std::string WideToUtf8(const wchar_t* wstr); -void WideToUtf8(const std::u16string_view& wstr, StringBuilderBase& out); void WideToUtf8(const std::wstring_view& wstr, StringBuilderBase& out); std::string WideToUtf8(const std::wstring_view Wstr); @@ -697,6 +696,19 @@ ForEachStrTok(const std::string_view& Str, char Delim, Fn&& Func) ////////////////////////////////////////////////////////////////////////// +inline int32_t +StrCaseCompare(const char* Lhs, const char* Rhs, int64_t Length=-1) +{ + // A helper for cross-platform case-insensitive string comparison. +#if ZEN_PLATFORM_WINDOWS + return (Length < 0) ? _stricmp(Lhs, Rhs) : _strnicmp(Lhs, Rhs, size_t(Length)); +#else + return (Length < 0) ? strcasecmp(Lhs, Rhs) : strncasecmp(Lhs, Rhs, size_t(Length)); +#endif +} + +////////////////////////////////////////////////////////////////////////// + /** * ASCII character bitset useful for fast and readable parsing * diff --git a/zencore/include/zencore/thread.h b/zencore/include/zencore/thread.h index 9fc4c87a2..3feb12936 100644 --- a/zencore/include/zencore/thread.h +++ b/zencore/include/zencore/thread.h @@ -143,7 +143,7 @@ public: ZENCORE_API bool Wait(int TimeoutMs = -1); ZENCORE_API void Terminate(int ExitCode); ZENCORE_API void Reset(); - inline [[nodiscard]] int Pid() const { return m_Pid; } + [[nodiscard]] inline int Pid() const { return m_Pid; } private: void* m_ProcessHandle = nullptr; @@ -174,6 +174,7 @@ private: ZENCORE_API bool IsProcessRunning(int pid); ZENCORE_API int GetCurrentProcessId(); +ZENCORE_API int GetCurrentThreadId(); ZENCORE_API void Sleep(int ms); diff --git a/zencore/include/zencore/uid.h b/zencore/include/zencore/uid.h index f4e9ab65a..d25aa8059 100644 --- a/zencore/include/zencore/uid.h +++ b/zencore/include/zencore/uid.h @@ -74,7 +74,7 @@ struct Oid size_t operator()(const Oid& id) const { const size_t seed = id.OidBits[0]; - return (seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1]) | (uint64_t(id.OidBits[2]) << 32); + return ((seed << 6) + (seed >> 2) + 0x9e3779b9 + uint64_t(id.OidBits[1])) | (uint64_t(id.OidBits[2]) << 32); } }; diff --git a/zencore/include/zencore/varint.h b/zencore/include/zencore/varint.h index 0c40dd66b..d7f2ebfb7 100644 --- a/zencore/include/zencore/varint.h +++ b/zencore/include/zencore/varint.h @@ -2,6 +2,8 @@ #include "intmath.h" +#include <algorithm> + namespace zen { // Variable-Length Integer Encoding diff --git a/zencore/include/zencore/windows.h b/zencore/include/zencore/windows.h index 68138566b..4149506e7 100644 --- a/zencore/include/zencore/windows.h +++ b/zencore/include/zencore/windows.h @@ -10,7 +10,12 @@ struct IUnknown; // Workaround for "combaseapi.h(229): error C2187: syntax erro #ifndef NOMINMAX # define NOMINMAX // We don't want your min/max macros #endif -#define WIN32_LEAN_AND_MEAN +#ifndef NOGDI +# define NOGDI // We don't want your GetObject define +#endif +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif #include <windows.h> #undef GetObject diff --git a/zencore/include/zencore/zencore.h b/zencore/include/zencore/zencore.h index d654770d0..6b98749fe 100644 --- a/zencore/include/zencore/zencore.h +++ b/zencore/include/zencore/zencore.h @@ -29,6 +29,18 @@ # define ZEN_PLATFORM_MACOS 1 #endif +#if ZEN_PLATFORM_WINDOWS +# if !defined(NOMINMAX) +# define NOMINMAX // stops Windows.h from defining 'min/max' macros +# endif +# if !defined(NOGDI) +# define NOGDI +# endif +# if !defined(WIN32_LEAN_AND_MEAN) +# define WIN32_LEAN_AND_MEAN // cut-down what Windows.h defines +# endif +#endif + ////////////////////////////////////////////////////////////////////////// // Compiler // @@ -66,7 +78,11 @@ #ifndef ZEN_THIRD_PARTY_INCLUDES_START # if ZEN_COMPILER_MSC -# define ZEN_THIRD_PARTY_INCLUDES_START __pragma(warning(push)) __pragma(warning(disable : 4668)) +# define ZEN_THIRD_PARTY_INCLUDES_START \ + __pragma(warning(push)) \ + __pragma(warning(disable : 4668)) /* use of undefined preprocessor macro */ \ + __pragma(warning(disable : 4267)) /* '=': conversion from 'size_t' to 'US' */ \ + __pragma(warning(disable : 4127)) # else # define ZEN_THIRD_PARTY_INCLUDES_START # endif @@ -110,6 +126,13 @@ #define ZEN_PLATFORM_SUPPORTS_UNALIGNED_LOADS 1 +#if defined(__SIZEOF_WCHAR_T__) && __SIZEOF_WCHAR_T__ == 4 +# define ZEN_SIZEOF_WCHAR_T 4 +#else + static_assert(sizeof(wchar_t) == 2, "wchar_t is expected to be two bytes in size"); +# define ZEN_SIZEOF_WCHAR_T 2 +#endif + ////////////////////////////////////////////////////////////////////////// // Assert // @@ -229,9 +252,15 @@ ZENCORE_API void zencore_forcelinktests(); ////////////////////////////////////////////////////////////////////////// #if ZEN_COMPILER_MSC -# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", off)) # define ZEN_ENABLE_OPTIMIZATION_ACTUAL __pragma(optimize("", on)) -#else +#elif ZEN_COMPILER_GCC +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC push_options" ) \ + _Pragma( "GCC optimize (\"O0\")" ) +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma( "GCC pop_options" ) +#elif ZEN_COMPILER_CLANG +# define ZEN_DISABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize off") +# define ZEN_ENABLE_OPTIMIZATION_ACTUAL _Pragma("clang optimize on") #endif // Set up optimization control macros, now that we have both the build settings and the platform macros diff --git a/zencore/iobuffer.cpp b/zencore/iobuffer.cpp index 7077942bf..fd652f580 100644 --- a/zencore/iobuffer.cpp +++ b/zencore/iobuffer.cpp @@ -391,18 +391,22 @@ IoBufferBuilder::ReadFromFileMaybe(IoBuffer& InBuffer) DWORD dwNumberOfBytesRead = 0; BOOL Success = ::ReadFile(FileRef.FileHandle, OutBuffer.MutableData(), DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); +#else + int Fd = int(intptr_t(FileRef.FileHandle)); + int Result = pread(Fd, OutBuffer.MutableData(), size_t(FileRef.FileChunkSize), off_t(FileRef.FileChunkOffset)); + bool Success = (Result < 0); + + uint32_t dwNumberOfBytesRead = uint32_t(Result); +#endif if (!Success) { ThrowLastError("ReadFile failed in IoBufferBuilder::ReadFromFileMaybe"); } - ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); + ZEN_ASSERT(dwNumberOfBytesRead == FileRef.FileChunkSize); return OutBuffer; -#else -# error Needs implementation -#endif } else { @@ -417,14 +421,14 @@ IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t } IoBuffer -IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) { uint64_t FileSize; #if ZEN_PLATFORM_WINDOWS CAtlFile DataFile; - HRESULT hRes = DataFile.Create(FileName, GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); + HRESULT hRes = DataFile.Create(FileName.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); if (FAILED(hRes)) { @@ -433,7 +437,7 @@ IoBufferBuilder::MakeFromFile(const path_char_t* FileName, uint64_t Offset, uint DataFile.GetSize((ULONGLONG&)FileSize); #else - int Fd = open(FileName, O_RDONLY); + int Fd = open(FileName.c_str(), O_RDONLY); if (Fd < 0) { return {}; diff --git a/zencore/memory.cpp b/zencore/memory.cpp index 14ea7ca1d..4053e85f9 100644 --- a/zencore/memory.cpp +++ b/zencore/memory.cpp @@ -1,5 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include <zencore/zencore.h> #include <zencore/intmath.h> #include <zencore/memory.h> #include <zencore/testing.h> diff --git a/zencore/stats.cpp b/zencore/stats.cpp index 0c0647999..1bb6f6de0 100644 --- a/zencore/stats.cpp +++ b/zencore/stats.cpp @@ -227,7 +227,7 @@ UniformSample::Snapshot() const uint64_t ValuesSize = Size(); std::vector<double> Values(ValuesSize); - for (int i = 0; i < ValuesSize; ++i) + for (int i = 0, n = int(ValuesSize); i < n; ++i) { Values[i] = double(m_Values[i]); } diff --git a/zencore/string.cpp b/zencore/string.cpp index b7670617f..35dfcfcf3 100644 --- a/zencore/string.cpp +++ b/zencore/string.cpp @@ -30,6 +30,17 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild } } +template<typename u32bit_iterator> +void +utf32to8_impl(u32bit_iterator StartIt, u32bit_iterator EndIt, ::zen::StringBuilderBase& OutString) +{ + for (; StartIt != EndIt; ++StartIt) + { + wchar_t cp = *StartIt; + OutString.AppendCodepoint(cp); + } +} + ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -76,7 +87,7 @@ FilepathFindExtension(const std::string_view& Path, const char* ExtensionToMatch // Look for extension introducer ('.') - for (size_t i = PathLen - 1; i >= 0; --i) + for (int64_t i = PathLen - 1; i >= 0; --i) { if (Path[i] == '.') return Path.data() + i; @@ -164,26 +175,24 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) void WideToUtf8(const wchar_t* Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, OutString); + WideToUtf8(std::wstring_view{Wstr}, OutString); } void WideToUtf8(const std::wstring_view& Wstr, StringBuilderBase& OutString) { - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, OutString); -} - -void -WideToUtf8(const std::u16string_view& Wstr, StringBuilderBase& OutString) -{ +#if ZEN_SIZEOF_WCHAR_T == 2 utf16to8_impl(begin(Wstr), end(Wstr), OutString); +#else + utf32to8_impl(begin(Wstr), end(Wstr), OutString); +#endif } std::string WideToUtf8(const wchar_t* Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr}, String); + WideToUtf8(std::wstring_view{Wstr}, String); return String.c_str(); } @@ -192,7 +201,7 @@ std::string WideToUtf8(const std::wstring_view Wstr) { ExtendableStringBuilder<128> String; - WideToUtf8(std::u16string_view{(char16_t*)Wstr.data(), Wstr.size()}, String); + WideToUtf8(std::wstring_view{Wstr.data(), Wstr.size()}, String); return String.c_str(); } @@ -942,6 +951,18 @@ TEST_CASE("string") CHECK_EQ(ToLower("TE%St"sv), "te%st"sv); } + SUBCASE("StrCaseCompare") + { + CHECK(StrCaseCompare("foo", "FoO") == 0); + CHECK(StrCaseCompare("Bar", "bAs") < 0); + CHECK(StrCaseCompare("bAr", "Bas") < 0); + CHECK(StrCaseCompare("BBr", "Bar") > 0); + CHECK(StrCaseCompare("Bbr", "BAr") > 0); + CHECK(StrCaseCompare("foo", "FoO", 3) == 0); + CHECK(StrCaseCompare("Bar", "bAs", 3) < 0); + CHECK(StrCaseCompare("BBr", "Bar", 2) > 0); + } + SUBCASE("ForEachStrTok") { const auto Tokens = "here,is,my,different,tokens"sv; diff --git a/zencore/thread.cpp b/zencore/thread.cpp index da711fe89..4c89feac4 100644 --- a/zencore/thread.cpp +++ b/zencore/thread.cpp @@ -4,10 +4,12 @@ #include <zencore/except.h> #include <zencore/string.h> +#include <zencore/testing.h> #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> #elif ZEN_PLATFORM_LINUX +# include <pthread.h> # include <unistd.h> #endif @@ -69,6 +71,8 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) std::string ThreadNameZ{ThreadName}; SetNameInternal(GetCurrentThreadId(), ThreadNameZ.c_str()); #else + std::string ThreadNameZ{ThreadName}; + pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); #endif } // namespace zen @@ -418,7 +422,9 @@ IsProcessRunning(int pid) return true; #else - ZEN_NOT_IMPLEMENTED(); + char Buffer[64]; + sprintf(Buffer, "/proc/%d", pid); + return access(Buffer, F_OK) == 0; #endif } @@ -428,7 +434,17 @@ GetCurrentProcessId() #if ZEN_PLATFORM_WINDOWS return ::GetCurrentProcessId(); #else - return getpid(); + return int(getpid()); +#endif +} + +int +GetCurrentThreadId() +{ +#if ZEN_PLATFORM_WINDOWS + return ::GetCurrentThreadId(); +#else + return int(gettid()); #endif } @@ -447,9 +463,20 @@ Sleep(int ms) // Testing related code follows... // +#if ZEN_WITH_TESTS + void thread_forcelink() { } +TEST_CASE("Thread") +{ + int Pid = GetCurrentProcessId(); + CHECK(Pid > 0); + CHECK(IsProcessRunning(Pid)); +} + +#endif // ZEN_WITH_TESTS + } // namespace zen diff --git a/zencore/xmake.lua b/zencore/xmake.lua index 5de7f476d..1abd86b7b 100644 --- a/zencore/xmake.lua +++ b/zencore/xmake.lua @@ -2,13 +2,24 @@ target('zencore') set_kind("static") add_files("**.cpp") add_includedirs("include", {public=true}) - add_includedirs("..\\3rdparty\\utfcpp\\source") - add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Win64", "$(projectdir)/3rdparty/Oodle/lib/Win64") + add_includedirs("$(projectdir)/3rdparty/utfcpp/source") + if is_os("windows") then + add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Win64") + add_linkdirs("$(projectdir)/3rdparty/Oodle/lib/Win64") + elseif is_os("linux") then + add_linkdirs("$(projectdir)/3rdparty/BLAKE3/lib/Linux_x64") + add_linkdirs("$(projectdir)/3rdparty/Oodle/lib/Linux_x64") + add_links("blake3") + add_links("oo2corelinux64") + add_links("pthread") + end add_packages( "vcpkg::spdlog", "vcpkg::fmt", "vcpkg::doctest", "vcpkg::lz4", + "vcpkg::mimalloc", + "vcpkg::json11", "vcpkg::cpr", "vcpkg::curl", -- required by cpr "vcpkg::zlib", -- required by curl diff --git a/zenhttp/httpasio.cpp b/zenhttp/httpasio.cpp index ea84e7e87..d94f6af2e 100644 --- a/zenhttp/httpasio.cpp +++ b/zenhttp/httpasio.cpp @@ -9,7 +9,9 @@ #include <memory_resource> ZEN_THIRD_PARTY_INCLUDES_START -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif #include <http_parser.h> #include <asio.hpp> ZEN_THIRD_PARTY_INCLUDES_END @@ -393,7 +395,7 @@ HttpServerConnection::EnqueueRead() ZEN_TRACE("read: conn#:{} seq#:{} t:{} bytes:{}", m_ConnectionId, m_RequestCounter.load(std::memory_order_relaxed), - GetCurrentThreadId(), + zen::GetCurrentThreadId(), ByteCount); OnDataReceived(Ec, ByteCount); @@ -1017,7 +1019,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode) m_Response.reset(new HttpResponse(HttpContentType::kBinary)); std::array<IoBuffer, 0> Empty; - m_Response->InitializeForPayload((UINT16)ResponseCode, Empty); + m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty); } void @@ -1026,7 +1028,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_ASSERT(!m_Response); m_Response.reset(new HttpResponse(ContentType)); - m_Response->InitializeForPayload((UINT16)ResponseCode, Blobs); + m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs); } void @@ -1157,6 +1159,13 @@ HttpAsioServer::Run(bool IsInteractive) { const bool TestMode = !IsInteractive; + int WaitTimeout = -1; + if (!TestMode) + { + WaitTimeout = 1000; + } + +#if ZEN_PLATFORM_WINDOWS if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (asio HTTP). Press ESC or Q to quit"); @@ -1164,13 +1173,6 @@ HttpAsioServer::Run(bool IsInteractive) do { - int WaitTimeout = -1; - - if (!TestMode) - { - WaitTimeout = 1000; - } - if (!TestMode && _kbhit() != 0) { char c = (char)_getch(); @@ -1183,6 +1185,18 @@ HttpAsioServer::Run(bool IsInteractive) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); +#else + if (TestMode == false) + { + zen::logging::ConsoleLog().info("Zen Server running (asio HTTP). Ctrl-C to quit"); + } + + do + { + m_ShutdownEvent.Wait(WaitTimeout); + } + while (!IsApplicationExitRequested()); +#endif } void diff --git a/zenhttp/httpclient.cpp b/zenhttp/httpclient.cpp index 6e915e613..e6813d407 100644 --- a/zenhttp/httpclient.cpp +++ b/zenhttp/httpclient.cpp @@ -22,7 +22,7 @@ using namespace std::literals; HttpClient::Response FromCprResponse(cpr::Response& InResponse) { - return {.StatusCode = InResponse.status_code}; + return {.StatusCode = int(InResponse.status_code)}; } ////////////////////////////////////////////////////////////////////////// @@ -130,7 +130,7 @@ HttpClient::TransactPackage(std::string_view Url, CbPackage Package) ResponseBuffer.SetContentType(ContentType); } - return {.StatusCode = FilterResponse.status_code, .ResponsePayload = ResponseBuffer}; + return {.StatusCode = int(FilterResponse.status_code), .ResponsePayload = ResponseBuffer}; } HttpClient::Response diff --git a/zenhttp/httpnull.cpp b/zenhttp/httpnull.cpp index e49051ac5..1a60e211c 100644 --- a/zenhttp/httpnull.cpp +++ b/zenhttp/httpnull.cpp @@ -2,9 +2,12 @@ #include "httpnull.h" -#include <conio.h> #include <zencore/logging.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif + namespace zen { HttpNullServer::HttpNullServer() @@ -32,6 +35,13 @@ HttpNullServer::Run(bool IsInteractiveSession) { const bool TestMode = !IsInteractiveSession; + int WaitTimeout = -1; + if (!TestMode) + { + WaitTimeout = 1000; + } + +#if ZEN_PLATFORM_WINDOWS if (TestMode == false) { zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Press ESC or Q to quit"); @@ -39,13 +49,6 @@ HttpNullServer::Run(bool IsInteractiveSession) do { - int WaitTimeout = -1; - - if (!TestMode) - { - WaitTimeout = 1000; - } - if (!TestMode && _kbhit() != 0) { char c = (char)_getch(); @@ -58,6 +61,18 @@ HttpNullServer::Run(bool IsInteractiveSession) m_ShutdownEvent.Wait(WaitTimeout); } while (!IsApplicationExitRequested()); +#else + if (TestMode == false) + { + zen::logging::ConsoleLog().info("Zen Server running (null HTTP). Ctrl-C to quit"); + } + + do + { + m_ShutdownEvent.Wait(WaitTimeout); + } + while (!IsApplicationExitRequested()); +#endif } void diff --git a/zenhttp/httpserver.cpp b/zenhttp/httpserver.cpp index f40836c4a..7b729cf0e 100644 --- a/zenhttp/httpserver.cpp +++ b/zenhttp/httpserver.cpp @@ -17,8 +17,6 @@ #include <zencore/thread.h> #include <zenhttp/httpshared.h> -#include <conio.h> -#include <new.h> #include <charconv> #include <mutex> #include <span> diff --git a/zenhttp/httpshared.cpp b/zenhttp/httpshared.cpp index c703409af..ab1463559 100644 --- a/zenhttp/httpshared.cpp +++ b/zenhttp/httpshared.cpp @@ -109,7 +109,7 @@ FormatPackageMessage(const CbPackage& Data) } } - return std::move(ResponseBuffers); + return ResponseBuffers; } CbPackage diff --git a/zenhttp/httpsys.cpp b/zenhttp/httpsys.cpp index cdf9e0a39..242954912 100644 --- a/zenhttp/httpsys.cpp +++ b/zenhttp/httpsys.cpp @@ -1221,7 +1221,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& const HTTP_REQUEST* HttpRequestPtr = Tx.HttpRequest(); const int PrefixLength = Service.UriPrefixLength(); - const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(char16_t); + const int AbsPathLength = HttpRequestPtr->CookedUrl.AbsPathLength / sizeof(wchar_t); HttpContentType AcceptContentType = HttpContentType::kUnknownContentType; @@ -1231,7 +1231,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& // with utf8. This is overhead which I'd prefer to avoid but for now we just have // to live with it - WideToUtf8({(char16_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, + WideToUtf8({(wchar_t*)HttpRequestPtr->CookedUrl.pAbsPath + PrefixLength, gsl::narrow<size_t>(AbsPathLength - PrefixLength)}, m_UriUtf8); std::string_view UriSuffix8{m_UriUtf8}; @@ -1268,7 +1268,7 @@ HttpSysServerRequest::HttpSysServerRequest(HttpSysTransaction& Tx, HttpService& { --QueryStringLength; // We skip the leading question mark - WideToUtf8({(char16_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(char16_t)}, m_QueryStringUtf8); + WideToUtf8({(wchar_t*)(HttpRequestPtr->CookedUrl.pQueryString) + 1, QueryStringLength / sizeof(wchar_t)}, m_QueryStringUtf8); } else { diff --git a/zenhttp/include/zenhttp/httpclient.h b/zenhttp/include/zenhttp/httpclient.h index 9ece86111..8316a9b9f 100644 --- a/zenhttp/include/zenhttp/httpclient.h +++ b/zenhttp/include/zenhttp/httpclient.h @@ -8,8 +8,6 @@ #include <zencore/uid.h> #include <zenhttp/httpcommon.h> -#include <zencore/windows.h> - ZEN_THIRD_PARTY_INCLUDES_START #include <cpr/cpr.h> ZEN_THIRD_PARTY_INCLUDES_END diff --git a/zenhttp/include/zenhttp/httpserver.h b/zenhttp/include/zenhttp/httpserver.h index 93ba452c7..614cdc2ac 100644 --- a/zenhttp/include/zenhttp/httpserver.h +++ b/zenhttp/include/zenhttp/httpserver.h @@ -46,7 +46,7 @@ public: if (Key.size() == ParamName.size()) { - if (0 == _strnicmp(Key.data(), ParamName.data(), Key.size())) + if (0 == StrCaseCompare(Key.data(), ParamName.data(), Key.size())) { return Kv.second; } diff --git a/zenhttp/include/zenhttp/httpshared.h b/zenhttp/include/zenhttp/httpshared.h index 2e728577d..dc660e45d 100644 --- a/zenhttp/include/zenhttp/httpshared.h +++ b/zenhttp/include/zenhttp/httpshared.h @@ -36,7 +36,7 @@ struct CbPackageHeader static_assert(sizeof(CbPackageHeader) == 16); -static constinit uint32_t kCbPkgMagic = 0xaa77aacc; +enum : uint32_t { kCbPkgMagic = 0xaa77aacc }; struct CbAttachmentEntry { diff --git a/zenhttp/workthreadpool.h b/zenhttp/workthreadpool.h index 6581cc08f..834339d50 100644 --- a/zenhttp/workthreadpool.h +++ b/zenhttp/workthreadpool.h @@ -6,7 +6,6 @@ #include <zencore/blockingqueue.h> #include <zencore/refcount.h> -#include <zencore/windows.h> #include <exception> #include <functional> diff --git a/zenserver-test/zenserver-test.cpp b/zenserver-test/zenserver-test.cpp index f3849b2cd..0fedd4b8b 100644 --- a/zenserver-test/zenserver-test.cpp +++ b/zenserver-test/zenserver-test.cpp @@ -1705,11 +1705,12 @@ TEST_CASE("zcache.policy") zen::IoHash Key; zen::IoHash PayloadId; - zen::CbPackage Package = GeneratePackage(Key, PayloadId); - auto Buf = ToBuffer(Package); // Store package locally { + zen::CbPackage Package = GeneratePackage(Key, PayloadId); + auto Buf = ToBuffer(Package); + CHECK(Package.GetAttachments().size() != 0); cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(LocalCfg.BaseUri, Bucket, Key)}, cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, @@ -1762,11 +1763,12 @@ TEST_CASE("zcache.policy") zen::IoHash Key; zen::IoHash PayloadId; - zen::CbPackage Package = GeneratePackage(Key, PayloadId); - auto Buf = ToBuffer(Package); // Store package upstream { + zen::CbPackage Package = GeneratePackage(Key, PayloadId); + auto Buf = ToBuffer(Package); + CHECK(Package.GetAttachments().size() != 0); cpr::Response Result = cpr::Put(cpr::Url{"{}/{}/{}"_format(UpstreamCfg.BaseUri, Bucket, Key)}, cpr::Body{(const char*)Buf.GetData(), Buf.GetSize()}, @@ -1928,7 +1930,7 @@ struct RemoteExecutionRequest if (auto It = m_Visit.m_HashToFile.find(NeedHash); It != m_Visit.m_HashToFile.end()) { - zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second.c_str()); + zen::IoBuffer FileData = zen::IoBufferBuilder::MakeFromFile(It->second); cpr::Response CasResponse = cpr::Post(cpr::Url(m_CasUri), cpr::Body((const char*)FileData.Data(), FileData.Size())); diff --git a/zenserver/admin/admin.cpp b/zenserver/admin/admin.cpp index 07211cbeb..44406fa70 100644 --- a/zenserver/admin/admin.cpp +++ b/zenserver/admin/admin.cpp @@ -1,7 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#pragma once - #include "admin.h" #include <zencore/compactbinarybuilder.h> diff --git a/zenserver/cache/structuredcache.cpp b/zenserver/cache/structuredcache.cpp index 35cb02cbb..0debea3c6 100644 --- a/zenserver/cache/structuredcache.cpp +++ b/zenserver/cache/structuredcache.cpp @@ -10,7 +10,7 @@ #include <zencore/stream.h> #include <zencore/timer.h> #include <zenhttp/httpserver.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include "monitoring/httpstats.h" #include "structuredcache.h" diff --git a/zenserver/cache/structuredcachestore.cpp b/zenserver/cache/structuredcachestore.cpp index e1d77a088..6f13ca1b0 100644 --- a/zenserver/cache/structuredcachestore.cpp +++ b/zenserver/cache/structuredcachestore.cpp @@ -3,7 +3,6 @@ #include "structuredcachestore.h" #include <zencore/except.h> -#include <zencore/windows.h> #include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> @@ -24,6 +23,10 @@ #include <zenstore/cidstore.h> #include <zenstore/gc.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <concepts> #include <filesystem> #include <ranges> diff --git a/zenserver/cache/structuredcachestore.h b/zenserver/cache/structuredcachestore.h index 760f4995c..ac5642a21 100644 --- a/zenserver/cache/structuredcachestore.h +++ b/zenserver/cache/structuredcachestore.h @@ -9,10 +9,9 @@ #include <zencore/uid.h> #include <zenstore/cas.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <compare> #include <filesystem> @@ -117,7 +116,9 @@ private: /** A cache bucket manages a single directory containing metadata and data for that bucket */ - struct CacheBucket; + struct CacheBucket + { + }; std::filesystem::path m_RootDir; RwLock m_Lock; diff --git a/zenserver/casstore.cpp b/zenserver/casstore.cpp index 88525bd36..872a40df8 100644 --- a/zenserver/casstore.cpp +++ b/zenserver/casstore.cpp @@ -51,6 +51,9 @@ HttpCasService::HttpCasService(CasStore& Store) : m_CasStore(Store) return ServerRequest.WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPut | HttpVerb::kHead); diff --git a/zenserver/compute/apply.cpp b/zenserver/compute/apply.cpp index 053c262c2..6669108d2 100644 --- a/zenserver/compute/apply.cpp +++ b/zenserver/compute/apply.cpp @@ -12,11 +12,11 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/scopeguard.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/cidstore.h> #if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> ZEN_THIRD_PARTY_INCLUDES_START # include <AccCtrl.h> # include <AclAPI.h> @@ -34,6 +34,7 @@ using namespace std::literals; namespace zen { +#if ZEN_PLATFORM_WINDOWS struct BasicFunctionJob { public: @@ -133,7 +134,11 @@ BasicFunctionJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +#endif + +//////////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS struct SandboxedFunctionJob { SandboxedFunctionJob() = default; @@ -323,6 +328,9 @@ SandboxedFunctionJob::SpawnJob(std::filesystem::path ExePath) return true; } +#endif + +//////////////////////////////////////////////////////////////////////////////// HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const std::filesystem::path& BaseDir) : m_Log(logging::Get("apply")) @@ -434,6 +442,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, SharedBuffer Decompressed = DataView.Decompress(); const uint64_t DecompressedSize = DataView.GetRawSize(); + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += DecompressedSize; ++AttachmentCount; @@ -464,9 +474,15 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -483,6 +499,9 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -580,6 +599,8 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, const IoHash DataHash = Attachment.GetHash(); CompressedBuffer DataView = Attachment.AsCompressedBinary(); + ZEN_UNUSED(DataHash); + const uint64_t CompressedSize = DataView.GetCompressedSize(); TotalAttachmentBytes += CompressedSize; @@ -605,8 +626,14 @@ HttpFunctionService::HttpFunctionService(CasStore& Store, CidStore& InCidStore, return HttpReq.WriteResponse(HttpResponseCode::OK, Output); } break; + + default: + break; } break; + + default: + break; } }, HttpVerb::kPost); @@ -743,6 +770,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) std::string_view ExecPath = Desc["path"].AsString(); std::filesystem::path ExePath = SandboxPath / ExecPath; +#if ZEN_PLATFORM_WINDOWS WideStringBuilder<512> CommandLine; CommandLine.Append(L'"'); CommandLine.Append(ExePath.c_str()); @@ -789,6 +817,7 @@ HttpFunctionService::ExecAction(const WorkerDesc& Worker, CbObject Action) GetExitCodeProcess(ProcessInformation.hProcess, &ExitCode); // Gather outputs +#endif // ZEN_PLATFORM_WINDOWS FileContents OutputData = zen::ReadFile(SandboxPath / "build.output"); diff --git a/zenserver/config.cpp b/zenserver/config.cpp index f512f8015..94226ef26 100644 --- a/zenserver/config.cpp +++ b/zenserver/config.cpp @@ -9,16 +9,16 @@ #include <zencore/string.h> #include <zenhttp/zenhttp.h> -#pragma warning(push) -#pragma warning(disable : 4267) // warning C4267: '=': conversion from 'size_t' to 'US', possible loss of data +ZEN_THIRD_PARTY_INCLUDES_START #include <cxxopts.hpp> -#pragma warning(pop) - #include <fmt/format.h> #include <zencore/logging.h> #include <sol/sol.hpp> +ZEN_THIRD_PARTY_INCLUDES_END -#include <conio.h> +#if ZEN_PLATFORM_WINDOWS +# include <conio.h> +#endif #if ZEN_PLATFORM_WINDOWS @@ -260,8 +260,13 @@ ParseGlobalCliOptions(int argc, char* argv[], ZenServerOptions& GlobalOptions, Z if (result.count("help")) { zen::logging::ConsoleLog().info("{}", options.help()); +#if ZEN_PLATFORM_WINDOWS zen::logging::ConsoleLog().info("Press any key to exit!"); _getch(); +#else + // Assume the user's in a terminal on all other platforms and that + // they'll use less/more/etc. if need be. +#endif exit(0); } @@ -286,7 +291,7 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv using namespace fmt::literals; std::filesystem::path ConfigScript = DataRoot / "zen_cfg.lua"; - zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript.native().c_str()); + zen::IoBuffer LuaScript = zen::IoBufferBuilder::MakeFromFile(ConfigScript); if (LuaScript) { @@ -301,6 +306,7 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv // any more than it needs to lua.set_function("getenv", [&](const std::string env) -> sol::object { +#if ZEN_PLATFORM_WINDOWS std::wstring EnvVarValue; size_t RequiredSize = 0; std::wstring EnvWide = zen::Utf8ToWide(env); @@ -312,6 +318,10 @@ ParseServiceConfig(const std::filesystem::path& DataRoot, ZenServiceConfig& Serv EnvVarValue.resize(RequiredSize); _wgetenv_s(&RequiredSize, EnvVarValue.data(), RequiredSize, EnvWide.c_str()); return sol::make_object(lua, zen::WideToUtf8(EnvVarValue.c_str())); +#else + ZEN_UNUSED(env); + return sol::make_object(lua, sol::lua_nil); +#endif }); try diff --git a/zenserver/config.h b/zenserver/config.h index 72a4f31bb..55e846352 100644 --- a/zenserver/config.h +++ b/zenserver/config.h @@ -4,6 +4,7 @@ #include <filesystem> #include <string> +#include <vector> #ifndef ZEN_ENABLE_MESH # define ZEN_ENABLE_MESH 0 diff --git a/zenserver/experimental/frontend.cpp b/zenserver/experimental/frontend.cpp index 98d570cfe..4bd3ec90a 100644 --- a/zenserver/experimental/frontend.cpp +++ b/zenserver/experimental/frontend.cpp @@ -54,7 +54,7 @@ body { <pre> __________ _________ __ \____ / ____ ____ / _____/_/ |_ ____ _______ ____ - / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ + / / _/ __ \ / \ \_____ \ \ __\ / _ \ \_ __ \_/ __ \ / /_ \ ___/ | | \ / \ | | ( <_> ) | | \/\ ___/ /_______ \ \___ >|___| //_______ / |__| \____/ |__| \___ > \/ \/ \/ \/ \/ diff --git a/zenserver/experimental/usnjournal.cpp b/zenserver/experimental/usnjournal.cpp index 9422dd485..95f75cea5 100644 --- a/zenserver/experimental/usnjournal.cpp +++ b/zenserver/experimental/usnjournal.cpp @@ -1,5 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#if ZEN_PLATFORM_WINDOWS + #include "usnjournal.h" #include <zencore/except.h> @@ -344,3 +346,5 @@ UsnJournalReader::Initialize(std::filesystem::path VolumePath) } } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/experimental/usnjournal.h b/zenserver/experimental/usnjournal.h index db1f59abc..2f666383b 100644 --- a/zenserver/experimental/usnjournal.h +++ b/zenserver/experimental/usnjournal.h @@ -2,6 +2,8 @@ #pragma once +#if ZEN_PLATFORM_WINDOWS + #include <zencore/windows.h> #include <zencore/zencore.h> @@ -63,3 +65,5 @@ private: }; } // namespace zen + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/projectstore.cpp b/zenserver/projectstore.cpp index 73d61c124..ba51a0dd2 100644 --- a/zenserver/projectstore.cpp +++ b/zenserver/projectstore.cpp @@ -11,11 +11,14 @@ #include <zencore/stream.h> #include <zencore/string.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #define USE_ROCKSDB 0 #if USE_ROCKSDB @@ -114,7 +117,7 @@ struct ProjectStore::OplogStorage : public RefCounted #if USE_ROCKSDB { - std::string RocksdbPath = WideToUtf8((m_OplogStoragePath / "ops.rdb").native().c_str()); + std::string RocksdbPath = ToUtf8(m_OplogStoragePath / "ops.rdb"); ZEN_DEBUG("opening rocksdb db at '{}'", RocksdbPath); @@ -184,8 +187,8 @@ struct ProjectStore::OplogStorage : public RefCounted CbObject Op(SharedBuffer::MakeView(OpBuffer.Data(), OpBuffer.Size())); m_NextOpsOffset = - Max(m_NextOpsOffset.load(std::memory_order::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); - m_MaxLsn = Max(m_MaxLsn.load(std::memory_order::memory_order_relaxed), LogEntry.OpLsn); + Max(m_NextOpsOffset.load(std::memory_order_relaxed), RoundUp(OpFileOffset + LogEntry.OpCoreSize, m_OpsAlign)); + m_MaxLsn = Max(m_MaxLsn.load(std::memory_order_relaxed), LogEntry.OpLsn); Handler(Op, LogEntry); }); @@ -275,8 +278,8 @@ private: ProjectStore::Oplog::Oplog(std::string_view Id, Project* Project, CidStore& Store, std::filesystem::path BasePath) : m_OuterProject(Project) , m_CidStore(Store) -, m_OplogId(Id) , m_BasePath(BasePath) +, m_OplogId(Id) { m_Storage = new OplogStorage(this, m_BasePath); const bool StoreExists = m_Storage->Exists(); @@ -331,7 +334,7 @@ ProjectStore::Oplog::FindChunk(Oid ChunkId) std::filesystem::path FilePath = m_OuterProject->RootDir / FileIt->second.ServerPath; - IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath.native().c_str()); + IoBuffer FileChunk = IoBufferBuilder::MakeFromFile(FilePath); FileChunk.SetContentType(ZenContentType::kBinary); return FileChunk; @@ -626,7 +629,7 @@ ProjectStore::Project::Write() CbObjectWriter Cfg; Cfg << "id" << Identifier; - Cfg << "root" << WideToUtf8(RootDir.c_str()); + Cfg << "root" << ToUtf8(RootDir); Cfg << "project" << ProjectRootDir; Cfg << "engine" << EngineRootDir; @@ -725,8 +728,6 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId) void ProjectStore::Project::DeleteOplog(std::string_view OplogId) { - bool Exists = false; - { RwLock::ExclusiveLockScope _(m_ProjectLock); @@ -734,8 +735,6 @@ ProjectStore::Project::DeleteOplog(std::string_view OplogId) if (OplogIt != m_Oplogs.end()) { - Exists = true; - m_Oplogs.erase(OplogIt); } } @@ -776,8 +775,8 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath) : m_Log(zen::logging::Get("project")) -, m_ProjectBasePath(BasePath) , m_CidStore(Store) +, m_ProjectBasePath(BasePath) { ZEN_INFO("initializing project store at '{}'", BasePath); // m_Log.set_level(spdlog::level::debug); @@ -1492,7 +1491,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) ProjectStore::Oplog& Log = *OplogIt; CbObjectWriter Cb; - Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempDir(); + Cb << "id"sv << Log.OplogId() << "project"sv << Prj.Identifier << "tempdir"sv << Log.TempPath().c_str(); Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cb.Save()); } @@ -1531,6 +1530,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::OK); } break; + + default: + break; } }, HttpVerb::kPost | HttpVerb::kGet | HttpVerb::kDelete); @@ -1628,7 +1630,7 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) const ProjectStore::Project& Prj = *ProjectIt; CbObjectWriter Response; - Response << "id" << Prj.Identifier << "root" << WideToUtf8(Prj.RootDir.c_str()); + Response << "id" << Prj.Identifier << "root" << ToUtf8(Prj.RootDir); Response.BeginArray("oplogs"sv); Prj.IterateOplogs([&](const ProjectStore::Oplog& I) { Response << "id"sv << I.OplogId(); }); @@ -1654,6 +1656,9 @@ HttpProjectService::HttpProjectService(CidStore& Store, ProjectStore* Projects) return Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kDelete); diff --git a/zenserver/projectstore.h b/zenserver/projectstore.h index c9f49217a..cebe26886 100644 --- a/zenserver/projectstore.h +++ b/zenserver/projectstore.h @@ -97,7 +97,6 @@ public: const std::string& OplogId() const { return m_OplogId; } - const std::wstring& TempDir() const { return m_TempPath.native(); } const std::filesystem::path& TempPath() const { return m_TempPath; } spdlog::logger& Log() { return m_OuterProject->Log(); } diff --git a/zenserver/testing/launch.cpp b/zenserver/testing/launch.cpp index 55695ac9c..569bf99e0 100644 --- a/zenserver/testing/launch.cpp +++ b/zenserver/testing/launch.cpp @@ -9,17 +9,21 @@ #include <zencore/iobuffer.h> #include <zencore/iohash.h> #include <zencore/logging.h> -#include <zencore/windows.h> -#include <zenstore/CAS.h> - -#include <AccCtrl.h> -#include <AclAPI.h> -#include <sddl.h> - -#include <UserEnv.h> -#pragma comment(lib, "UserEnv.lib") +#include <zenstore/cas.h> + +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> + +ZEN_THIRD_PARTY_INCLUDES_START +# include <AccCtrl.h> +# include <AclAPI.h> +# include <atlbase.h> +# include <sddl.h> +# include <UserEnv.h> +ZEN_THIRD_PARTY_INCLUDES_END +# pragma comment(lib, "UserEnv.lib") +#endif -#include <atlbase.h> #include <filesystem> #include <span> @@ -27,6 +31,7 @@ using namespace std::literals; namespace zen { +#if ZEN_PLATFORM_WINDOWS struct BasicJob { public: @@ -126,7 +131,19 @@ BasicJob::ExitCode() return gsl::narrow_cast<int>(Ec); } +#else + struct BasicJob + { + void SetWorkingDirectory(...); + void SpawnJob(...); + void Wait(); + int32_t ExitCode(); + }; +#endif // ZEN_PLATFORM_WINDOWS +//////////////////////////////////////////////////////////////////////////////// + +#if ZEN_PLATFORM_WINDOWS struct SandboxedJob { SandboxedJob() = default; @@ -316,6 +333,10 @@ SandboxedJob::SpawnJob(std::filesystem::path ExePath) return true; } +#else +#endif // ZEN_PLATFORM_WINDOWS + +//////////////////////////////////////////////////////////////////////////////// HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::path& SandboxBaseDir) : m_Log(logging::Get("exec")) @@ -336,6 +357,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat case HttpVerb::kPost: break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -362,6 +386,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat Job.SpawnJob("c:\\windows\\system32\\cmd.exe"); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); @@ -413,6 +440,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response); } break; + + default: + break; } }, HttpVerb::kPost); @@ -496,6 +526,9 @@ HttpLaunchService::HttpLaunchService(CasStore& Store, const std::filesystem::pat return HttpReq.WriteResponse(HttpResponseCode::OK, Response.Save()); } break; + + default: + break; } }, HttpVerb::kGet | HttpVerb::kPost); diff --git a/zenserver/upstream/upstreamcache.cpp b/zenserver/upstream/upstreamcache.cpp index 168449d05..db1981ec7 100644 --- a/zenserver/upstream/upstreamcache.cpp +++ b/zenserver/upstream/upstreamcache.cpp @@ -185,7 +185,7 @@ namespace detail { if (CacheRecord.Type == ZenContentType::kBinary) { CloudCacheResult Result; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { if (m_UseLegacyDdc) { @@ -211,7 +211,7 @@ namespace detail { for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) { if (CloudCacheResult Result = Session.PutCompressedBlob(CacheRecord.PayloadIds[Idx], Payloads[Idx]); Result.Success) @@ -233,7 +233,7 @@ namespace detail { } Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts; Attempt++) { if (PutRefResult Result = Session.PutRef(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, @@ -512,7 +512,7 @@ namespace detail { Package.Save(MemStream); IoBuffer PackagePayload(IoBuffer::Wrap, MemStream.Data(), MemStream.Size()); - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, @@ -530,7 +530,7 @@ namespace detail { for (size_t Idx = 0, Count = Payloads.size(); Idx < Count; Idx++) { Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCachePayload(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, @@ -553,7 +553,7 @@ namespace detail { } Result.Success = false; - for (int32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) + for (uint32_t Attempt = 0; Attempt < MaxAttempts && !Result.Success; Attempt++) { Result = Session.PutCacheRecord(CacheRecord.CacheKey.Bucket, CacheRecord.CacheKey.Hash, RecordValue, CacheRecord.Type); diff --git a/zenserver/upstream/zen.h b/zenserver/upstream/zen.h index 12e46bd8d..8e81d1cb6 100644 --- a/zenserver/upstream/zen.h +++ b/zenserver/upstream/zen.h @@ -10,14 +10,14 @@ #include <zencore/uid.h> #include <zencore/zencore.h> -#pragma warning(push) -#pragma warning(disable : 4127) +ZEN_THIRD_PARTY_INCLUDES_START #include <tsl/robin_map.h> -#pragma warning(pop) +ZEN_THIRD_PARTY_INCLUDES_END #include <asio.hpp> #include <chrono> +#include <list> struct ZenCacheValue; diff --git a/zenserver/windows/service.cpp b/zenserver/windows/service.cpp index 23cefb7b5..0eec01698 100644 --- a/zenserver/windows/service.cpp +++ b/zenserver/windows/service.cpp @@ -2,6 +2,8 @@ #include "service.h" +#if ZEN_PLATFORM_WINDOWS + #include <zencore/except.h> #include <zencore/zencore.h> @@ -638,3 +640,5 @@ SvcReportEvent(LPTSTR szFunction) // DeregisterEventSource(hEventSource); //} } + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/zenserver/zenserver.cpp b/zenserver/zenserver.cpp index 9a8090fc0..f5c38baae 100644 --- a/zenserver/zenserver.cpp +++ b/zenserver/zenserver.cpp @@ -12,13 +12,16 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/timer.h> -#include <zencore/windows.h> #include <zenhttp/httpserver.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/cidstore.h> #include <zenutil/zenserverprocess.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #if ZEN_USE_MIMALLOC ZEN_THIRD_PARTY_INCLUDES_START # include <mimalloc-new-delete.h> @@ -384,7 +387,7 @@ public: for (auto& PidEntry : m_ServerEntry->SponsorPids) { - if (uint32_t ThisPid = PidEntry.load(std::memory_order::memory_order_relaxed)) + if (uint32_t ThisPid = PidEntry.load(std::memory_order_relaxed)) { if (PidEntry.compare_exchange_strong(ThisPid, 0)) { diff --git a/zenstore/basicfile.cpp b/zenstore/basicfile.cpp index 9ed70a5ec..938c0f9e2 100644 --- a/zenstore/basicfile.cpp +++ b/zenstore/basicfile.cpp @@ -9,6 +9,14 @@ #include <zencore/testing.h> #include <zencore/testutils.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#else +# include <fcntl.h> +# include <sys/file.h> +# include <sys/stat.h> +#endif + #include <fmt/format.h> #include <gsl/gsl-lite.hpp> @@ -38,6 +46,7 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& { Ec.clear(); +#if ZEN_PLATFORM_WINDOWS const DWORD dwCreationDisposition = IsCreate ? CREATE_ALWAYS : OPEN_EXISTING; DWORD dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; const DWORD dwShareMode = FILE_SHARE_READ; @@ -63,6 +72,19 @@ BasicFile::Open(std::filesystem::path FileName, bool IsCreate, std::error_code& return; } +#else + int OpenFlags = O_RDWR; + OpenFlags |= IsCreate ? O_CREAT|O_TRUNC : 0; + + int Fd = open(FileName.c_str(), OpenFlags, 0666); + if (Fd < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + return; + } + + void* FileHandle = (void*)(uintptr_t(Fd)); +#endif m_FileHandle = FileHandle; } @@ -72,7 +94,12 @@ BasicFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS ::CloseHandle(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + close(Fd); +#endif m_FileHandle = nullptr; } } @@ -86,6 +113,7 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) { const uint64_t NumberOfBytesToRead = Min(BytesToRead, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -95,6 +123,12 @@ BasicFile::Read(void* Data, uint64_t BytesToRead, uint64_t FileOffset) BOOL Success = ::ReadFile(m_FileHandle, Data, DWORD(NumberOfBytesToRead), &dwNumberOfBytesRead, &Ovl); ZEN_ASSERT(dwNumberOfBytesRead == NumberOfBytesToRead); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesRead = pread(Fd, Data, NumberOfBytesToRead, FileOffset); + bool Success = (BytesRead > 0); +#endif if (!Success) { @@ -161,6 +195,7 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro { const uint64_t NumberOfBytesToWrite = Min(Size, MaxChunkSize); +#if ZEN_PLATFORM_WINDOWS OVERLAPPED Ovl{}; Ovl.Offset = DWORD(FileOffset & 0xffff'ffffu); @@ -169,6 +204,12 @@ BasicFile::Write(const void* Data, uint64_t Size, uint64_t FileOffset, std::erro DWORD dwNumberOfBytesWritten = 0; BOOL Success = ::WriteFile(m_FileHandle, Data, DWORD(NumberOfBytesToWrite), &dwNumberOfBytesWritten, &Ovl); +#else + static_assert(sizeof(off_t) >= sizeof(uint64_t), "sizeof(off_t) does not support large files"); + int Fd = int(uintptr_t(m_FileHandle)); + int BytesWritten = pwrite(Fd, Data, NumberOfBytesToWrite, FileOffset); + bool Success = (BytesWritten > 0); +#endif if (!Success) { @@ -210,16 +251,29 @@ BasicFile::WriteAll(IoBuffer Data, std::error_code& Ec) void BasicFile::Flush() { +#if ZEN_PLATFORM_WINDOWS FlushFileBuffers(m_FileHandle); +#else + int Fd = int(uintptr_t(m_FileHandle)); + fsync(Fd); +#endif } uint64_t BasicFile::FileSize() { +#if ZEN_PLATFORM_WINDOWS ULARGE_INTEGER liFileSize; liFileSize.LowPart = ::GetFileSize(m_FileHandle, &liFileSize.HighPart); return uint64_t(liFileSize.QuadPart); +#else + int Fd = int(uintptr_t(m_FileHandle)); + static_assert(sizeof(decltype(stat::st_size)) == sizeof(uint64_t), "fstat() doesn't support large files"); + struct stat Stat; + fstat(Fd, &Stat); + return uint64_t(Stat.st_size); +#endif } ////////////////////////////////////////////////////////////////////////// @@ -234,11 +288,16 @@ TemporaryFile::Close() { if (m_FileHandle) { +#if ZEN_PLATFORM_WINDOWS // Mark file for deletion when final handle is closed FILE_DISPOSITION_INFO Fdi{.DeleteFile = TRUE}; SetFileInformationByHandle(m_FileHandle, FileDispositionInfo, &Fdi, sizeof Fdi); +#else + std::filesystem::path FilePath = zen::PathFromHandle(m_FileHandle); + unlink(FilePath.c_str()); +#endif BasicFile::Close(); } @@ -280,6 +339,7 @@ LockFile::~LockFile() void LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_code& Ec) { +#if ZEN_PLATFORM_WINDOWS Ec.clear(); const DWORD dwCreationDisposition = CREATE_ALWAYS; @@ -302,6 +362,26 @@ LockFile::Create(std::filesystem::path FileName, CbObject Payload, std::error_co return; } +#elif ZEN_PLATFORM_LINUX + int Fd = open(FileName.c_str(), O_RDWR|O_CREAT, 0666); + if (Fd < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + return; + } + + int LockRet = flock(Fd, LOCK_EX); + if (LockRet < 0) + { + Ec = zen::MakeErrorCodeFromLastError(); + close(Fd); + return; + } + + void* FileHandle = (void*)uintptr_t(Fd); +#else +# error check flock() support +#endif m_FileHandle = FileHandle; diff --git a/zenstore/CAS.cpp b/zenstore/cas.cpp index a4bbfa340..4268e314b 100644 --- a/zenstore/CAS.cpp +++ b/zenstore/cas.cpp @@ -180,7 +180,7 @@ CasImpl::Initialize(const CasStoreConfiguration& InConfig) id.ToString(manifest); Marker.Open(ManifestPath.c_str(), /* IsCreate */ true); - Marker.Write(manifest.c_str(), (DWORD)manifest.Size(), 0); + Marker.Write(manifest.c_str(), uint32_t(manifest.Size()), 0); } } diff --git a/zenstore/caslog.cpp b/zenstore/caslog.cpp index 2bac6affd..ae2ea973b 100644 --- a/zenstore/caslog.cpp +++ b/zenstore/caslog.cpp @@ -2,7 +2,7 @@ #include <zenstore/cas.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -121,7 +121,7 @@ CasLogFile::Replay(std::function<void(const void*)>&& Handler) m_File.Read(ReadBuffer.data(), LogDataSize, LogBaseOffset); - for (int i = 0; i < LogEntryCount; ++i) + for (int i = 0; i < int(LogEntryCount); ++i) { Handler(ReadBuffer.data() + (i * m_RecordSize)); } @@ -135,7 +135,7 @@ CasLogFile::Append(const void* DataPointer, uint64_t DataSize) uint64_t AppendOffset = m_AppendOffset.fetch_add(DataSize); std::error_code Ec; - m_File.Write(DataPointer, gsl::narrow<DWORD>(DataSize), AppendOffset, Ec); + m_File.Write(DataPointer, gsl::narrow<uint32_t>(DataSize), AppendOffset, Ec); if (Ec) { diff --git a/zenstore/cidstore.cpp b/zenstore/cidstore.cpp index 7a5d7bcf4..c67269e0d 100644 --- a/zenstore/cidstore.cpp +++ b/zenstore/cidstore.cpp @@ -5,7 +5,7 @@ #include <zencore/compress.h> #include <zencore/filesystem.h> #include <zencore/logging.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/caslog.h> #include <filesystem> diff --git a/zenstore/compactcas.cpp b/zenstore/compactcas.cpp index 612f87c7c..f80bc85c4 100644 --- a/zenstore/compactcas.cpp +++ b/zenstore/compactcas.cpp @@ -2,7 +2,7 @@ #include <zenstore/cas.h> -#include "CompactCas.h" +#include "compactcas.h" #include <zencore/except.h> #include <zencore/logging.h> diff --git a/zenstore/compactcas.h b/zenstore/compactcas.h index a512c3d93..a23b80828 100644 --- a/zenstore/compactcas.h +++ b/zenstore/compactcas.h @@ -9,11 +9,14 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> #include <zenstore/caslog.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + namespace zen { ////////////////////////////////////////////////////////////////////////// diff --git a/zenstore/filecas.cpp b/zenstore/filecas.cpp index a37450cd8..6d385a208 100644 --- a/zenstore/filecas.cpp +++ b/zenstore/filecas.cpp @@ -1,6 +1,6 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include "FileCas.h" +#include "filecas.h" #include <zencore/except.h> #include <zencore/filesystem.h> @@ -23,7 +23,9 @@ #include <unordered_map> ZEN_THIRD_PARTY_INCLUDES_START -#include <atlfile.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlfile.h> +#endif ZEN_THIRD_PARTY_INCLUDES_END namespace zen { @@ -33,7 +35,6 @@ using namespace fmt::literals; FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash) { ShardedPath.Append(RootPath.c_str()); - ShardedPath.Append(std::filesystem::path::preferred_separator); ExtendableStringBuilder<64> HashString; ChunkHash.ToHexString(HashString); @@ -53,13 +54,14 @@ FileCasStrategy::ShardingHelper::ShardingHelper(const std::filesystem::path& Roo // would probably be a good idea to measure performance for different // policies and chunk counts + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str, str + 3); - ShardedPath.Append(std::filesystem::path::preferred_separator); + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str + 3, str + 5); Shard2len = ShardedPath.Size(); - ShardedPath.Append(std::filesystem::path::preferred_separator); + ShardedPath.AppendSeparator(); ShardedPath.AppendAsciiRange(str + 5, str + 40); } @@ -84,6 +86,9 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) { ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash); + RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); + +#if ZEN_PLATFORM_WINDOWS const HANDLE ChunkFileHandle = FileRef.FileHandle; auto DeletePayloadFileOnClose = [&] { @@ -104,8 +109,6 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) // // Future improvement: maintain Bloom filter to avoid expensive file system probes? - RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); - { CAtlFile PayloadFile; @@ -224,6 +227,49 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash) ChunkHash); DeletePayloadFileOnClose(); +#elif ZEN_PLATFORM_LINUX + std::filesystem::path SourcePath = PathFromHandle(FileRef.FileHandle); + std::filesystem::path DestPath = Name.ShardedPath.c_str(); + int Ret = link(SourcePath.c_str(), DestPath.c_str()); + if (Ret < 0 && zen::GetLastError() == ENOENT) + { + // Destination directory doesn't exist. Create it any try again. + CreateDirectories(DestPath.parent_path().c_str()); + Ret = link(SourcePath.c_str(), DestPath.c_str()); + } + int LinkError = zen::GetLastError(); + + // Unlink the file. If the path to unlink didn't exist someone else + // beat us to it and that is hunky-dory. + if (unlink(SourcePath.c_str()) < 0) + { + int UnlinkError = zen::GetLastError(); + if (UnlinkError != ENOENT) + { + ZEN_WARN("unlink of CAS payload file failed ('{}')", GetSystemErrorAsString(UnlinkError)); + } + } + + // It is possible that someone beat us to it in linking the file. In that + // case a "file exists" error is okay. All others are not. + if (Ret < 0) + { + if (LinkError == EEXIST) + { + return CasStore::InsertResult{.New = false}; + } + + ZEN_WARN("link of CAS payload file failed ('{}'), falling back to regular write for insert of {}", + GetSystemErrorAsString(LinkError), + ChunkHash); + } + else + { + return CasStore::InsertResult{.New = true}; + } +#else +# error check link/unlink for this platform +#endif // ZEN_PLATFORM_* } return InsertChunk(Chunk.Data(), Chunk.Size(), ChunkHash); @@ -238,6 +284,7 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize // // Future improvement: maintain Bloom filter to avoid expensive file system probes? +#if ZEN_PLATFORM_WINDOWS CAtlFile PayloadFile; HRESULT hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); @@ -250,9 +297,18 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize } PayloadFile.Close(); +#elif ZEN_PLATFORM_LINUX + if (access(Name.ShardedPath.c_str(), F_OK) == 0) + { + return CasStore::InsertResult{.New = false}; + } +#else +# error Check access() for this platform +#endif RwLock::ExclusiveLockScope _(LockForHash(ChunkHash)); +#if ZEN_PLATFORM_WINDOWS // For now, use double-checked locking to see if someone else was first hRes = PayloadFile.Create(Name.ShardedPath.c_str(), GENERIC_READ, FILE_SHARE_READ, OPEN_EXISTING); @@ -286,6 +342,42 @@ FileCasStrategy::InsertChunk(const void* const ChunkData, const size_t ChunkSize { ThrowSystemException(hRes, "Failed to open shard file '{}'"_format(WideToUtf8(Name.ShardedPath))); } +#else + // Attempt to exclusively create the file. + auto InternalCreateFile = [&] { return open(Name.ShardedPath.c_str(), O_WRONLY|O_CREAT|O_EXCL, 0666); }; + int Fd = InternalCreateFile(); + if (Fd < 0) + { + switch (zen::GetLastError()) + { + case EEXIST: + // Another thread has beat us to it so we're golden. + return {.New = false}; + + case ENOENT: + if (zen::CreateDirectories(std::string_view(Name.ShardedPath.c_str(), Name.Shard2len))) + { + Fd = InternalCreateFile(); + if (Fd >= 0) + { + break; + } + } + ThrowLastError("Failed creating shard directory '{}'"_format(Name.ShardedPath)); + + default: + ThrowLastError("Unexpected error occurred opening shard file '{}'"_format(Name.ShardedPath)); + } + } + + struct FdWrapper + { + ~FdWrapper() { Close(); } + void Write(const void* Cursor, size_t Size) { (void)!write(Fd, Cursor, Size); } + void Close() { if (Fd >= 0) { close(Fd); Fd = -1; } } + int Fd; + } PayloadFile = { Fd }; +#endif // ZEN_PLATFORM_WINDOWS size_t ChunkRemain = ChunkSize; auto ChunkCursor = reinterpret_cast<const uint8_t*>(ChunkData); @@ -337,7 +429,7 @@ FileCasStrategy::DeleteChunk(const IoHash& ChunkHash, std::error_code& Ec) { ShardingHelper Name(m_Config.RootDirectory.c_str(), ChunkHash); - ZEN_DEBUG("deleting CAS payload file '{}'", WideToUtf8(Name.ShardedPath)); + ZEN_DEBUG("deleting CAS payload file '{}'", Name.ShardedPath.ToUtf8()); std::filesystem::remove(Name.ShardedPath.c_str(), Ec); } @@ -362,13 +454,13 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& struct Visitor : public FileSystemTraversal::TreeVisitor { Visitor(const std::filesystem::path& RootDir) : RootDirectory(RootDir) {} - virtual void VisitFile(const std::filesystem::path& Parent, const std::wstring_view& File, uint64_t FileSize) override + virtual void VisitFile(const std::filesystem::path& Parent, const path_view& File, uint64_t FileSize) override { ZEN_UNUSED(FileSize); std::filesystem::path RelPath = std::filesystem::relative(Parent, RootDirectory); - std::wstring PathString = RelPath.native(); + std::filesystem::path::string_type PathString = RelPath.native(); if ((PathString.size() == (3 + 2 + 1)) && (File.size() == (40 - 3 - 2))) { @@ -378,12 +470,15 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& } PathString.append(File); - StringBuilder<64> Utf8; - WideToUtf8(PathString, Utf8); - // TODO: should validate that we're actually dealing with a valid hex string here +#if ZEN_PLATFORM_WINDOWS + StringBuilder<64> Utf8; + WideToUtf8(PathString, Utf8); IoHash NameHash = IoHash::FromHexString({Utf8.Data(), Utf8.Size()}); +#else + IoHash NameHash = IoHash::FromHexString(PathString); +#endif BasicFile PayloadFile; std::error_code Ec; @@ -397,7 +492,7 @@ FileCasStrategy::IterateChunks(std::function<void(const IoHash& Hash, BasicFile& } virtual bool VisitDirectory([[maybe_unused]] const std::filesystem::path& Parent, - [[maybe_unused]] const std::wstring_view& DirectoryName) + [[maybe_unused]] const path_view& DirectoryName) { return true; } diff --git a/zenstore/filecas.h b/zenstore/filecas.h index 14314ce52..a86232588 100644 --- a/zenstore/filecas.h +++ b/zenstore/filecas.h @@ -4,9 +4,9 @@ #include <zencore/zencore.h> +#include <zencore/filesystem.h> #include <zencore/iobuffer.h> #include <zencore/iohash.h> -#include <zencore/string.h> #include <zencore/thread.h> #include <zenstore/cas.h> @@ -52,8 +52,8 @@ private: { ShardingHelper(const std::filesystem::path& RootPath, const IoHash& ChunkHash); - size_t Shard2len = 0; - ExtendableWideStringBuilder<128> ShardedPath; + size_t Shard2len = 0; + ExtendablePathBuilder<128> ShardedPath; }; }; diff --git a/zenstore/include/zenstore/basicfile.h b/zenstore/include/zenstore/basicfile.h index e4414787c..2df016c76 100644 --- a/zenstore/include/zenstore/basicfile.h +++ b/zenstore/include/zenstore/basicfile.h @@ -5,7 +5,6 @@ #include "zenstore.h" #include <zencore/iobuffer.h> -#include <zencore/windows.h> #include <filesystem> #include <functional> diff --git a/zenstore/include/zenstore/CAS.h b/zenstore/include/zenstore/cas.h index 86e7e78d9..7f8b5bfa2 100644 --- a/zenstore/include/zenstore/CAS.h +++ b/zenstore/include/zenstore/cas.h @@ -41,9 +41,9 @@ public: void AddChunksToSet(std::span<const IoHash> HashesToAdd); void RemoveChunksIf(std::function<bool(const IoHash& CandidateHash)>&& Predicate); void IterateChunks(std::function<void(const IoHash& ChunkHash)>&& Callback); - inline [[nodiscard]] bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } - inline [[nodiscard]] bool IsEmpty() const { return m_ChunkSet.empty(); } - inline [[nodiscard]] size_t GetSize() const { return m_ChunkSet.size(); } + [[nodiscard]] inline bool ContainsChunk(const IoHash& Hash) const { return m_ChunkSet.find(Hash) != m_ChunkSet.end(); } + [[nodiscard]] inline bool IsEmpty() const { return m_ChunkSet.empty(); } + [[nodiscard]] inline size_t GetSize() const { return m_ChunkSet.size(); } private: // Q: should we protect this with a lock, or is that a higher level concern? diff --git a/zenstore/include/zenstore/caslog.h b/zenstore/include/zenstore/caslog.h index 00b987383..1c6ee0bf6 100644 --- a/zenstore/include/zenstore/caslog.h +++ b/zenstore/include/zenstore/caslog.h @@ -8,10 +8,13 @@ #include <zencore/string.h> #include <zencore/thread.h> #include <zencore/uid.h> -#include <zencore/windows.h> #include <zenstore/basicfile.h> #include <zenstore/cas.h> +#if ZEN_PLATFORM_WINDOWS +# include <zencore/windows.h> +#endif + #include <functional> namespace zen { diff --git a/zenstore/include/zenstore/cidstore.h b/zenstore/include/zenstore/cidstore.h index 5f567e7fc..871ac553e 100644 --- a/zenstore/include/zenstore/cidstore.h +++ b/zenstore/include/zenstore/cidstore.h @@ -6,11 +6,9 @@ #include <tsl/robin_map.h> #include <zencore/iohash.h> -#include <zenstore/CAS.h> +#include <zenstore/cas.h> -namespace std::filesystem { -class path; -} +#include <filesystem> namespace zen { diff --git a/zenstore/zenstore.cpp b/zenstore/zenstore.cpp index d852fa64b..cc4a34fdb 100644 --- a/zenstore/zenstore.cpp +++ b/zenstore/zenstore.cpp @@ -2,7 +2,7 @@ #include "zenstore/zenstore.h" -#include <zenstore/CAS.h> +#include <zenstore/cas.h> #include <zenstore/basicfile.h> #include "filecas.h" diff --git a/zenstore/zenstore.vcxproj b/zenstore/zenstore.vcxproj index eb2ecd02b..9431b7b11 100644 --- a/zenstore/zenstore.vcxproj +++ b/zenstore/zenstore.vcxproj @@ -28,7 +28,7 @@ <ClInclude Include="include\zenstore\cidstore.h" /> <ClInclude Include="include\zenstore\gc.h" /> <ClInclude Include="include\zenstore\scrub.h" /> - <ClInclude Include="include\zenstore\CAS.h" /> + <ClInclude Include="include\zenstore\cas.h" /> <ClInclude Include="include\zenstore\caslog.h" /> <ClInclude Include="include\zenstore\zenstore.h" /> </ItemGroup> diff --git a/zenstore/zenstore.vcxproj.filters b/zenstore/zenstore.vcxproj.filters index 8a52c69f6..904de0748 100644 --- a/zenstore/zenstore.vcxproj.filters +++ b/zenstore/zenstore.vcxproj.filters @@ -14,7 +14,7 @@ <ItemGroup> <ClInclude Include="compactcas.h" /> <ClInclude Include="filecas.h" /> - <ClInclude Include="include\zenstore\CAS.h" /> + <ClInclude Include="include\zenstore\cas.h" /> <ClInclude Include="include\zenstore\caslog.h" /> <ClInclude Include="include\zenstore\gc.h" /> <ClInclude Include="include\zenstore\scrub.h" /> diff --git a/zenutil/zenserverprocess.cpp b/zenutil/zenserverprocess.cpp index 4098954a8..7cdf884fd 100644 --- a/zenutil/zenserverprocess.cpp +++ b/zenutil/zenserverprocess.cpp @@ -8,18 +8,25 @@ #include <zencore/logging.h> #include <zencore/session.h> #include <zencore/string.h> +#include <zencore/thread.h> -#include <atlbase.h> -#include <shellapi.h> +#include <atomic> #include <source_location> -#include <zencore/windows.h> +#if ZEN_PLATFORM_WINDOWS +# include <atlbase.h> +# include <shellapi.h> +# include <zencore/windows.h> +#else +# include <sys/mman.h> +#endif ////////////////////////////////////////////////////////////////////////// namespace zen { namespace zenutil { +#if ZEN_PLATFORM_WINDOWS class SecurityAttributes { public: @@ -53,6 +60,7 @@ namespace zenutil { } } }; +#endif // ZEN_PLATFORM_WINDOWS } // namespace zenutil @@ -72,59 +80,85 @@ ZenServerState::~ZenServerState() m_OurEntry = nullptr; } +#if ZEN_PLATFORM_WINDOWS if (m_Data) { UnmapViewOfFile(m_Data); - m_Data = nullptr; } if (m_hMapFile) { CloseHandle(m_hMapFile); } +#else + if (m_Data != nullptr) + { + munmap(m_Data, m_MaxEntryCount * sizeof(ZenServerEntry)); + } + + int Fd = int(intptr_t(m_hMapFile)); + close(Fd); +#endif + + m_Data = nullptr; } void ZenServerState::Initialize() { + size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); + +#if ZEN_PLATFORM_WINDOWS // TODO: there's a small chance of a race here, this logic could be tightened up with a mutex to // ensure only a single process at a time creates the mapping - if (HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap")) - { - m_hMapFile = hMap; - } - else + HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); + if (hMap == NULL) { // Security attributes to enable any user to access state zenutil::AnyUserSecurityAttributes Attrs; - hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file - Attrs.Attributes(), // allow anyone to access - PAGE_READWRITE, // read/write access - 0, // maximum object size (high-order DWORD) - m_MaxEntryCount * sizeof(ZenServerEntry), // maximum object size (low-order DWORD) - L"Global\\ZenMap"); // name of mapping object + hMap = CreateFileMapping(INVALID_HANDLE_VALUE, // use paging file + Attrs.Attributes(), // allow anyone to access + PAGE_READWRITE, // read/write access + 0, // maximum object size (high-order DWORD) + MapSize, // maximum object size (low-order DWORD) + L"Global\\ZenMap"); // name of mapping object if (hMap == NULL) { ThrowLastError("Could not open or create file mapping object for Zen server state"); } - - m_hMapFile = hMap; } - void* pBuf = MapViewOfFile(m_hMapFile, // handle to map object + void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_ALL_ACCESS, // read/write permission 0, // offset high 0, // offset low - m_MaxEntryCount * sizeof(ZenServerEntry)); + MapSize); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } +#else + int Fd = shm_open("UnrealEngineZen", O_RDWR|O_CREAT, 0666); + if (Fd < 0) + { + ThrowLastError("Could not open a shared memory object"); + } + void* hMap = (void*)intptr_t(Fd); + + ftruncate(Fd, MapSize); + + void* pBuf = mmap(nullptr, MapSize, PROT_READ|PROT_WRITE, MAP_SHARED, Fd, 0); + if (pBuf == MAP_FAILED) + { + ThrowLastError("Could not map view of Zen server state"); + } +#endif + m_hMapFile = hMap; m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); m_IsReadOnly = false; } @@ -132,27 +166,42 @@ ZenServerState::Initialize() bool ZenServerState::InitializeReadOnly() { - if (HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap")) - { - m_hMapFile = hMap; - } - else + size_t MapSize = m_MaxEntryCount * sizeof(ZenServerEntry); + +#if ZEN_PLATFORM_WINDOWS + HANDLE hMap = OpenFileMapping(FILE_MAP_ALL_ACCESS, FALSE, L"Global\\ZenMap"); + if (hMap == NULL) { return false; } - void* pBuf = MapViewOfFile(m_hMapFile, // handle to map object + void* pBuf = MapViewOfFile(hMap, // handle to map object FILE_MAP_READ, // read permission 0, // offset high 0, // offset low - m_MaxEntryCount * sizeof(ZenServerEntry)); + MapSize); if (pBuf == NULL) { ThrowLastError("Could not map view of Zen server state"); } +#else + int Fd = shm_open("UnrealEngineZen", O_RDONLY, 0666); + if (Fd < 0) + { + return false; + } + void* hMap = (void*)intptr_t(Fd); + + void* pBuf = mmap(nullptr, MapSize, PROT_READ, MAP_PRIVATE, Fd, 0); + if (pBuf == MAP_FAILED) + { + ThrowLastError("Could not map read-only view of Zen server state"); + } +#endif - m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); + m_hMapFile = hMap; + m_Data = reinterpret_cast<ZenServerEntry*>(pBuf); return true; } @@ -187,7 +236,7 @@ ZenServerState::Register(int ListenPort) { ZenServerEntry& Entry = m_Data[i]; - if (Entry.ListenPort.load(std::memory_order::memory_order_relaxed) == 0) + if (Entry.ListenPort.load(std::memory_order_relaxed) == 0) { uint16_t Expected = 0; if (Entry.ListenPort.compare_exchange_strong(Expected, uint16_t(ListenPort))) @@ -280,7 +329,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd) { for (std::atomic<uint32_t>& PidEntry : SponsorPids) { - if (PidEntry.load(std::memory_order::memory_order_relaxed) == 0) + if (PidEntry.load(std::memory_order_relaxed) == 0) { uint32_t Expected = 0; if (PidEntry.compare_exchange_strong(Expected, uint16_t(PidToAdd))) @@ -289,7 +338,7 @@ ZenServerState::ZenServerEntry::AddSponsorProcess(uint32_t PidToAdd) return true; } } - else if (PidEntry.load(std::memory_order::memory_order_relaxed) == PidToAdd) + else if (PidEntry.load(std::memory_order_relaxed) == PidToAdd) { // Success, the because pid is already in the list return true; @@ -404,12 +453,13 @@ ZenServerInstance::Shutdown() void ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerArgs) { +#if ZEN_PLATFORM_WINDOWS ZEN_ASSERT(!m_Process.IsValid()); // Only spawn once const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = BaseDir / "zenserver.exe"; - const int MyPid = _getpid(); + const int MyPid = zen::GetCurrentProcessId(); const int ChildId = ++ChildIdCounter; ExtendableStringBuilder<32> ChildEventName; @@ -555,6 +605,9 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr } m_ReadyEvent = std::move(ChildEvent); +#else + ZEN_UNUSED(BasePort, AdditionalServerArgs); +#endif // ZEN_PLATFORM_WINDOWS } void |