diff options
| author | Stefan Boberg <[email protected]> | 2026-03-16 10:56:58 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-03-16 10:56:58 +0100 |
| commit | 7079dcf0d77f1ff4cfe66f87f21710068223a8d0 (patch) | |
| tree | 69f4b4ffea7da9b02e5cbf23b74c19558027007e /src | |
| parent | Fix thread vector growth in dynamic thread pools (diff) | |
| parent | Enable cross compilation of Windows targets on Linux (#839) (diff) | |
| download | zen-7079dcf0d77f1ff4cfe66f87f21710068223a8d0.tar.xz zen-7079dcf0d77f1ff4cfe66f87f21710068223a8d0.zip | |
Merge branch 'main' into sb/threadpool
Diffstat (limited to 'src')
35 files changed, 973 insertions, 480 deletions
diff --git a/src/transports/winsock/xmake.lua b/src/transports/winsock/xmake.lua index c14283546..cdba75885 100644 --- a/src/transports/winsock/xmake.lua +++ b/src/transports/winsock/xmake.lua @@ -5,7 +5,7 @@ target("winsock") set_group("transports") add_headerfiles("**.h") add_files("**.cpp") - add_links("Ws2_32") + add_links("ws2_32") add_includedirs(".") set_symbols("debug") add_deps("zenbase", "transport-sdk") diff --git a/src/zen/cmds/service_cmd.cpp b/src/zen/cmds/service_cmd.cpp index a781dc340..3347f1afe 100644 --- a/src/zen/cmds/service_cmd.cpp +++ b/src/zen/cmds/service_cmd.cpp @@ -12,8 +12,8 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> # include <shellapi.h> -# include <Shlwapi.h> -# pragma comment(lib, "Shlwapi.lib") +# include <shlwapi.h> +# pragma comment(lib, "shlwapi.lib") #endif #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua index 4c134404a..df249ade4 100644 --- a/src/zen/xmake.lua +++ b/src/zen/xmake.lua @@ -14,8 +14,10 @@ target("zen") if is_plat("windows") then add_files("zen.rc") - add_ldflags("/subsystem:console,5.02") - add_ldflags("/LTCG") + add_ldflags("/subsystem:console,5.02", {force = true}) + if not (get_config("toolchain") or ""):find("clang") then + add_ldflags("/LTCG") + end end if is_plat("macosx") then diff --git a/src/zen/zen.rc b/src/zen/zen.rc index 0617681a7..3adf25b72 100644 --- a/src/zen/zen.rc +++ b/src/zen/zen.rc @@ -7,7 +7,7 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #pragma code_page(1252) -101 ICON "..\\zen.ico" +101 ICON "../zen.ico" VS_VERSION_INFO VERSIONINFO FILEVERSION ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER,0 diff --git a/src/zenbase/include/zenbase/zenbase.h b/src/zenbase/include/zenbase/zenbase.h index 2aec1f314..1d5051c5b 100644 --- a/src/zenbase/include/zenbase/zenbase.h +++ b/src/zenbase/include/zenbase/zenbase.h @@ -211,7 +211,24 @@ char (&ZenArrayCountHelper(const T (&)[N]))[N + 1]; # define ZEN_EXE_SUFFIX_LITERAL "" #endif -#define ZEN_UNUSED(...) ((void)__VA_ARGS__) +#if ZEN_COMPILER_CLANG +// Clang warns about the comma operator in ((void)a, b) with -Wunused-value. +// Use a fold expression via a helper to suppress each argument individually. +namespace zen::detail { +inline void +unused_impl() +{ +} +template<typename... T> +inline void +unused_impl(T&&...) +{ +} +} // namespace zen::detail +# define ZEN_UNUSED(...) ::zen::detail::unused_impl(__VA_ARGS__) +#else +# define ZEN_UNUSED(...) ((void)__VA_ARGS__) +#endif ////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore-test/targetver.h b/src/zencore-test/targetver.h index d432d6993..4805141de 100644 --- a/src/zencore-test/targetver.h +++ b/src/zencore-test/targetver.h @@ -7,4 +7,4 @@ // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. -#include <SDKDDKVer.h> +#include <sdkddkver.h> diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index ee0b0625a..a16bb3f13 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -6,7 +6,7 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> -# include <Dbghelp.h> +# include <DbgHelp.h> #endif #if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 8ed63565c..0d361801f 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -17,7 +17,7 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> -# include <ShlObj.h> +# include <shlobj.h> # pragma comment(lib, "shell32.lib") # pragma comment(lib, "ole32.lib") #endif @@ -32,17 +32,27 @@ ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_PLATFORM_LINUX # include <dirent.h> # include <fcntl.h> +# include <linux/fs.h> +# include <linux/magic.h> +# include <sys/ioctl.h> # include <sys/resource.h> # include <sys/mman.h> # include <sys/stat.h> +# include <sys/vfs.h> # include <pwd.h> # include <unistd.h> +// XFS_SUPER_MAGIC is not always defined in linux/magic.h +# ifndef XFS_SUPER_MAGIC +# define XFS_SUPER_MAGIC 0x58465342 +# endif #endif #if ZEN_PLATFORM_MAC # include <dirent.h> # include <fcntl.h> # include <libproc.h> +# include <sys/attr.h> +# include <sys/clonefile.h> # include <sys/resource.h> # include <sys/mman.h> # include <sys/stat.h> @@ -59,6 +69,53 @@ namespace zen { using namespace std::literals; +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +struct ScopedFd +{ + int Fd = -1; + + ScopedFd() = default; + explicit ScopedFd(int InFd) : Fd(InFd) {} + + ~ScopedFd() + { + if (Fd >= 0) + { + close(Fd); + } + } + + ScopedFd(const ScopedFd&) = delete; + ScopedFd& operator=(const ScopedFd&) = delete; + + ScopedFd(ScopedFd&& Other) noexcept : Fd(Other.Fd) { Other.Fd = -1; } + + ScopedFd& operator=(ScopedFd&& Other) noexcept + { + if (this != &Other) + { + if (Fd >= 0) + { + close(Fd); + } + Fd = Other.Fd; + Other.Fd = -1; + } + return *this; + } + + // Release ownership of the file descriptor, returning it without closing + int Release() + { + int Result = Fd; + Fd = -1; + return Result; + } + + explicit operator bool() const { return Fd >= 0; } +}; +#endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC + #if ZEN_PLATFORM_WINDOWS static bool @@ -615,6 +672,38 @@ SupportsBlockRefCounting(std::filesystem::path Path) } return true; +#elif ZEN_PLATFORM_LINUX + struct statfs Buf; + if (statfs(Path.c_str(), &Buf) != 0) + { + return false; + } + + // Btrfs and XFS (when formatted with reflink support) support FICLONE + return Buf.f_type == BTRFS_SUPER_MAGIC || Buf.f_type == XFS_SUPER_MAGIC; +#elif ZEN_PLATFORM_MAC + struct attrlist AttrList = {}; + AttrList.bitmapcount = ATTR_BIT_MAP_COUNT; + AttrList.volattr = ATTR_VOL_CAPABILITIES; + + struct + { + uint32_t Length; + vol_capabilities_attr_t Capabilities; + } AttrBuf = {}; + + if (getattrlist(Path.c_str(), &AttrList, &AttrBuf, sizeof(AttrBuf), 0) != 0) + { + return false; + } + + // Check that the VOL_CAP_INT_CLONE bit is both valid and set + if (!(AttrBuf.Capabilities.valid[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_CLONE)) + { + return false; + } + + return !!(AttrBuf.Capabilities.capabilities[VOL_CAPABILITIES_INTERFACES] & VOL_CAP_INT_CLONE); #else ZEN_UNUSED(Path); return false; @@ -768,7 +857,115 @@ private: DWORD m_TargetVolumeSerialNumber; }; -#endif // ZEN_PLATFORM_WINDOWS +#elif ZEN_PLATFORM_LINUX + +class LinuxCloneQueryInterface : public CloneQueryInterface +{ +public: + LinuxCloneQueryInterface(uint64_t AlignmentSize, dev_t TargetDevice) : m_AlignmentSize(AlignmentSize), m_TargetDevice(TargetDevice) {} + + virtual bool CanClone(void* SourceNativeHandle) override + { + int Fd = int(uintptr_t(SourceNativeHandle)); + + struct stat St; + if (fstat(Fd, &St) != 0) + { + return false; + } + + // Source must be on the same filesystem as the target + return St.st_dev == m_TargetDevice; + } + + virtual uint64_t GetClonableRange(uint64_t SourceOffset, + uint64_t TargetOffset, + uint64_t Size, + uint64_t& OutPreBytes, + uint64_t& OutPostBytes) override + { + if (Size < m_AlignmentSize) + { + return 0; + } + + uint64_t PreBytes = (m_AlignmentSize - (SourceOffset % m_AlignmentSize)) % m_AlignmentSize; + uint64_t PostBytes = (SourceOffset + Size) % m_AlignmentSize; + ZEN_ASSERT(Size >= PreBytes + PostBytes); + if (Size - (PreBytes + PostBytes) < m_AlignmentSize) + { + return 0; + } + ZEN_ASSERT((PreBytes < Size && PostBytes < Size && Size >= PreBytes + PostBytes + m_AlignmentSize)); + + const uint64_t DestCloneOffset = TargetOffset + PreBytes; + if (DestCloneOffset % m_AlignmentSize != 0) + { + return 0; + } + + OutPreBytes = PreBytes; + OutPostBytes = PostBytes; + uint64_t CloneSize = Size - (PreBytes + PostBytes); + ZEN_ASSERT(CloneSize % m_AlignmentSize == 0); + return CloneSize; + } + + virtual bool TryClone(void* SourceNativeHandle, + void* TargetNativeHandle, + uint64_t AlignedSourceOffset, + uint64_t AlignedTargetOffset, + uint64_t AlignedSize, + uint64_t TargetFinalSize) override + { + ZEN_ASSERT_SLOW(CanClone(SourceNativeHandle)); + ZEN_ASSERT((AlignedSourceOffset % m_AlignmentSize) == 0); + ZEN_ASSERT((AlignedTargetOffset % m_AlignmentSize) == 0); + ZEN_ASSERT(AlignedSize > 0); + ZEN_ASSERT((AlignedSize % m_AlignmentSize) == 0); + + int SourceFd = int(uintptr_t(SourceNativeHandle)); + int TargetFd = int(uintptr_t(TargetNativeHandle)); + + // Ensure the target file is sized to its final size before cloning + struct stat TargetSt; + if (fstat(TargetFd, &TargetSt) != 0 || uint64_t(TargetSt.st_size) != TargetFinalSize) + { + if (ftruncate(TargetFd, TargetFinalSize) != 0) + { + std::error_code DummyEc; + ZEN_DEBUG("Failed setting final size {} for file {}", TargetFinalSize, PathFromHandle(TargetNativeHandle, DummyEc)); + return false; + } + } + + struct file_clone_range Range = {}; + Range.src_fd = SourceFd; + Range.src_offset = AlignedSourceOffset; + Range.src_length = AlignedSize; + Range.dest_offset = AlignedTargetOffset; + + if (ioctl(TargetFd, FICLONERANGE, &Range) != 0) + { + std::error_code DummyEc; + ZEN_DEBUG("Failed cloning {} bytes from file {} at {} to file {} at {}", + AlignedSize, + PathFromHandle(SourceNativeHandle, DummyEc), + AlignedSourceOffset, + PathFromHandle(TargetNativeHandle, DummyEc), + AlignedTargetOffset); + return false; + } + + return true; + } + +private: + uint64_t m_AlignmentSize; + dev_t m_TargetDevice; +}; + +#endif // ZEN_PLATFORM_WINDOWS / ZEN_PLATFORM_LINUX std::unique_ptr<CloneQueryInterface> GetCloneQueryInterface(const std::filesystem::path& TargetDirectory) @@ -819,7 +1016,30 @@ GetCloneQueryInterface(const std::filesystem::path& TargetDirectory) return std::make_unique<WindowsCloneQueryInterface>(SectorsPerCluster * BytesPerSector, DestVolumeSerialNumber); } } -#else // ZEN_PLATFORM_WINDOWS +#elif ZEN_PLATFORM_LINUX + struct statfs FsBuf; + if (statfs(TargetDirectory.c_str(), &FsBuf) != 0) + { + ZEN_DEBUG("Failed to get filesystem info for path {}", TargetDirectory); + return {}; + } + + // Only Btrfs and XFS support FICLONERANGE + if (FsBuf.f_type != BTRFS_SUPER_MAGIC && FsBuf.f_type != XFS_SUPER_MAGIC) + { + return {}; + } + + struct stat StBuf; + if (stat(TargetDirectory.c_str(), &StBuf) != 0) + { + ZEN_DEBUG("Failed to stat path {}", TargetDirectory); + return {}; + } + + uint64_t AlignmentSize = FsBuf.f_bsize; + return std::make_unique<LinuxCloneQueryInterface>(AlignmentSize, StBuf.st_dev); +#else ZEN_UNUSED(TargetDirectory); #endif // ZEN_PLATFORM_WINDOWS return {}; @@ -1000,40 +1220,44 @@ TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& return TryCloneFile((void*)FromFile.m_Handle, (void*)TargetFile.m_Handle); #elif ZEN_PLATFORM_LINUX -# if 0 - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - // The 'from' file - int FromFd = open(FromPath.c_str(), O_RDONLY|O_CLOEXEC); - if (FromFd < 0) + ScopedFd FromFd(open(FromPath.c_str(), O_RDONLY | O_CLOEXEC)); + if (!FromFd) { return false; } - ScopedFd $From = { FromFd }; + + // Remove any existing target so we can create a fresh clone + unlink(ToPath.c_str()); // The 'to' file - int ToFd = open(ToPath.c_str(), O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC, 0666); - if (ToFd < 0) + ScopedFd ToFd(open(ToPath.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0666)); + if (!ToFd) { return false; } - fchmod(ToFd, 0666); - ScopedFd $To = { ToFd }; - ioctl(ToFd, FICLONE, FromFd); + if (ioctl(ToFd.Fd, FICLONE, FromFd.Fd) != 0) + { + // Clone not supported by this filesystem or files are on different volumes. + // Remove the empty target file we created. + ToFd = ScopedFd(); + unlink(ToPath.c_str()); + return false; + } - return false; -# endif // 0 - ZEN_UNUSED(FromPath, ToPath); - return false; + return true; #elif ZEN_PLATFORM_MAC - /* clonefile() syscall if APFS */ - ZEN_UNUSED(FromPath, ToPath); - return false; + // Remove any existing target - clonefile() requires the destination not exist + unlink(ToPath.c_str()); + + if (clonefile(FromPath.c_str(), ToPath.c_str(), CLONE_NOFOLLOW) != 0) + { + // Clone not supported (non-APFS) or files are on different volumes + return false; + } + + return true; #endif // ZEN_PLATFORM_WINDOWS } @@ -1069,9 +1293,7 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP if (Options.MustClone) { -#if ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX - ZEN_ERROR("CloneFile() is not implemented on this platform"); -#endif // ZEN_PLATFORM_MAC || ZEN_PLATFORM_LINUX + ZEN_ERROR("CloneFile() failed for {} -> {}", FromPath, ToPath); return false; } @@ -1084,35 +1306,27 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP &CancelFlag, /* dwCopyFlags */ 0); #else - struct ScopedFd - { - ~ScopedFd() { close(Fd); } - int Fd; - }; - // From file - int FromFd = open(FromPath.c_str(), O_RDONLY | O_CLOEXEC); - if (FromFd < 0) + ScopedFd FromFd(open(FromPath.c_str(), O_RDONLY | O_CLOEXEC)); + if (!FromFd) { ThrowLastError(fmt::format("failed to open file {}", FromPath)); } - ScopedFd $From = {FromFd}; // To file - int ToFd = open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); - if (ToFd < 0) + ScopedFd ToFd(open(ToPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666)); + if (!ToFd) { ThrowLastError(fmt::format("failed to create file {}", ToPath)); } - fchmod(ToFd, 0666); - ScopedFd $To = {ToFd}; + fchmod(ToFd.Fd, 0666); struct stat Stat; - fstat(FromFd, &Stat); + fstat(FromFd.Fd, &Stat); size_t FileSizeBytes = Stat.st_size; - int $Ignore = fchown(ToFd, Stat.st_uid, Stat.st_gid); + int $Ignore = fchown(ToFd.Fd, Stat.st_uid, Stat.st_gid); ZEN_UNUSED($Ignore); // What's the appropriate error handling here? // Copy impl @@ -1120,14 +1334,14 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP void* Buffer = malloc(BufferSize); while (true) { - int BytesRead = read(FromFd, Buffer, BufferSize); + int BytesRead = read(FromFd.Fd, Buffer, BufferSize); if (BytesRead <= 0) { Success = (BytesRead == 0); break; } - if (write(ToFd, Buffer, BytesRead) != BytesRead) + if (write(ToFd.Fd, Buffer, BytesRead) != BytesRead) { Success = false; break; @@ -1371,20 +1585,20 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer } #else - int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; - int Fd = open(Path.c_str(), OpenFlags, 0666); - if (Fd < 0) + int OpenFlags = O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC; + ScopedFd OutFd(open(Path.c_str(), OpenFlags, 0666)); + if (!OutFd) { zen::CreateDirectories(Path.parent_path()); - Fd = open(Path.c_str(), OpenFlags, 0666); + OutFd = ScopedFd(open(Path.c_str(), OpenFlags, 0666)); } - if (Fd < 0) + if (!OutFd) { ThrowLastError(fmt::format("File open failed for '{}'", Path)); } - fchmod(Fd, 0666); + fchmod(OutFd.Fd, 0666); #endif // TODO: this should be block-enlightened @@ -1408,9 +1622,9 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else - if (write(Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) + if (write(OutFd.Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) { - close(Fd); + OutFd = ScopedFd(); std::error_code DummyEc; RemoveFile(Path, DummyEc); ThrowLastError(fmt::format("File write failed for '{}'", Path)); @@ -1424,8 +1638,6 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer #if ZEN_PLATFORM_WINDOWS Outfile.Close(); -#else - close(Fd); #endif } @@ -1707,8 +1919,8 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi ProcessFunc(ReadBuffer.data(), dwBytesRead); } #else - int Fd = open(Path.c_str(), O_RDONLY | O_CLOEXEC); - if (Fd < 0) + ScopedFd InFd(open(Path.c_str(), O_RDONLY | O_CLOEXEC)); + if (!InFd) { return false; } @@ -1718,7 +1930,7 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi void* Buffer = malloc(ChunkSize); while (true) { - int BytesRead = read(Fd, Buffer, ChunkSize); + int BytesRead = read(InFd.Fd, Buffer, ChunkSize); if (BytesRead < 0) { Success = false; @@ -1734,7 +1946,6 @@ ScanFile(std::filesystem::path Path, const uint64_t ChunkSize, std::function<voi } free(Buffer); - close(Fd); if (!Success) { @@ -3123,28 +3334,26 @@ public: ZEN_UNUSED(SystemGlobal); std::string InstanceMapName = fmt::format("/{}", Name); - int Fd = shm_open(InstanceMapName.c_str(), O_RDWR, 0666); - if (Fd < 0) + ScopedFd FdGuard(shm_open(InstanceMapName.c_str(), O_RDWR, 0666)); + if (!FdGuard) { return {}; } - void* hMap = (void*)intptr_t(Fd); struct stat Stat; - fstat(Fd, &Stat); + fstat(FdGuard.Fd, &Stat); if (size_t(Stat.st_size) < Size) { - close(Fd); return {}; } - void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, FdGuard.Fd, 0); if (pBuf == MAP_FAILED) { - close(Fd); return {}; } + void* hMap = (void*)intptr_t(FdGuard.Release()); return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC } @@ -3199,23 +3408,22 @@ public: ZEN_UNUSED(SystemGlobal); std::string InstanceMapName = fmt::format("/{}", Name); - int Fd = shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); - if (Fd < 0) + ScopedFd FdGuard(shm_open(InstanceMapName.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666)); + if (!FdGuard) { return {}; } - fchmod(Fd, 0666); - void* hMap = (void*)intptr_t(Fd); + fchmod(FdGuard.Fd, 0666); - int Result = ftruncate(Fd, Size); + int Result = ftruncate(FdGuard.Fd, Size); ZEN_UNUSED(Result); - void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, Fd, 0); + void* pBuf = mmap(nullptr, Size, PROT_READ | PROT_WRITE, MAP_SHARED, FdGuard.Fd, 0); if (pBuf == MAP_FAILED) { - close(Fd); return {}; } + void* hMap = (void*)intptr_t(FdGuard.Release()); return Data{.Handle = hMap, .DataPtr = pBuf, .Size = Size, .Name = std::string(Name)}; #endif // ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC } @@ -3590,6 +3798,241 @@ TEST_CASE("RotateDirectories") } } +TEST_CASE("TryCloneFile") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".clone_test"; + CleanDirectory(TestBaseDir, true); + + SUBCASE("clone produces identical content") + { + std::filesystem::path SrcPath = TestBaseDir / "src.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst.bin"; + + // Write source file with known content + const char Content[] = "Hello, clone world! This is test data for TryCloneFile."; + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, Content, sizeof(Content))); + CHECK(IsFile(SrcPath)); + + bool Cloned = TryCloneFile(SrcPath, DstPath); + + if (Cloned) + { + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(DstContents.Data[0].GetSize(), sizeof(Content)); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + else + { + // Clone not supported on this filesystem - that's okay, just verify it didn't leave debris + ZEN_INFO("TryCloneFile not supported on this filesystem, skipping content check"); + } + } + + SUBCASE("clone overwrites existing target") + { + std::filesystem::path SrcPath = TestBaseDir / "src_overwrite.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst_overwrite.bin"; + + const char OldContent[] = "old content"; + const char NewContent[] = "new content that is longer than the old one"; + WriteFile(DstPath, IoBuffer(IoBuffer::Wrap, OldContent, sizeof(OldContent))); + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, NewContent, sizeof(NewContent))); + + bool Cloned = TryCloneFile(SrcPath, DstPath); + + if (Cloned) + { + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(NewContent)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), NewContent, sizeof(NewContent)), 0); + } + } + + SUBCASE("clone of nonexistent source fails") + { + std::filesystem::path SrcPath = TestBaseDir / "no_such_file.bin"; + std::filesystem::path DstPath = TestBaseDir / "dst_nosrc.bin"; + + CHECK_FALSE(TryCloneFile(SrcPath, DstPath)); + CHECK_FALSE(IsFile(DstPath)); + } + + DeleteDirectories(TestBaseDir); +} + +TEST_CASE("CopyFile.Clone") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".copyfile_clone_test"; + CleanDirectory(TestBaseDir, true); + + const char Content[] = "CopyFile clone test content with some bytes to verify integrity."; + std::filesystem::path SrcPath = TestBaseDir / "src.bin"; + WriteFile(SrcPath, IoBuffer(IoBuffer::Wrap, Content, sizeof(Content))); + + SUBCASE("EnableClone copies file regardless of clone support") + { + std::filesystem::path DstPath = TestBaseDir / "dst_enable.bin"; + + CopyFileOptions Options; + Options.EnableClone = true; + bool Success = CopyFile(SrcPath, DstPath, Options); + CHECK(Success); + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + + SUBCASE("DisableClone still copies file") + { + std::filesystem::path DstPath = TestBaseDir / "dst_disable.bin"; + + CopyFileOptions Options; + Options.EnableClone = false; + bool Success = CopyFile(SrcPath, DstPath, Options); + CHECK(Success); + CHECK(IsFile(DstPath)); + CHECK_EQ(FileSizeFromPath(DstPath), sizeof(Content)); + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + CHECK_EQ(memcmp(DstContents.Data[0].Data(), Content, sizeof(Content)), 0); + } + + DeleteDirectories(TestBaseDir); +} + +TEST_CASE("SupportsBlockRefCounting") +{ + std::filesystem::path BinDir = GetRunningExecutablePath().parent_path(); + + // Should not crash or throw on a valid path + bool Supported = SupportsBlockRefCounting(BinDir); + ZEN_INFO("SupportsBlockRefCounting({}) = {}", BinDir, Supported); + + // Should return false for nonexistent path + CHECK_FALSE(SupportsBlockRefCounting("/no/such/path/anywhere")); +} + +TEST_CASE("CloneQueryInterface") +{ + std::filesystem::path TestBaseDir = GetRunningExecutablePath().parent_path() / ".clonequery_test"; + CleanDirectory(TestBaseDir, true); + + auto CloneQuery = GetCloneQueryInterface(TestBaseDir); + + if (CloneQuery) + { + ZEN_INFO("CloneQueryInterface available for {}", TestBaseDir); + + // Write a source file large enough to exercise alignment + const uint64_t FileSize = 256 * 1024; + IoBuffer SrcBuf(FileSize); + { + uint8_t* Ptr = SrcBuf.MutableData<uint8_t>(); + for (uint64_t i = 0; i < FileSize; i++) + { + Ptr[i] = uint8_t(i * 37 + 7); + } + } + + std::filesystem::path SrcPath = TestBaseDir / "clone_src.bin"; + std::filesystem::path DstPath = TestBaseDir / "clone_dst.bin"; + WriteFile(SrcPath, SrcBuf); + + // Open source and target as native handles +# if ZEN_PLATFORM_WINDOWS + windows::Handle SrcHandle(CreateFileW(SrcPath.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + 0, + nullptr)); + CHECK(SrcHandle != INVALID_HANDLE_VALUE); + void* SrcNativeHandle = (void*)SrcHandle.m_Handle; + + windows::Handle DstHandle( + CreateFileW(DstPath.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, nullptr, OPEN_ALWAYS, 0, nullptr)); + CHECK(DstHandle != INVALID_HANDLE_VALUE); + void* DstNativeHandle = (void*)DstHandle.m_Handle; +# else + ScopedFd SrcFd(open(SrcPath.c_str(), O_RDONLY | O_CLOEXEC)); + CHECK(bool(SrcFd)); + void* SrcNativeHandle = (void*)uintptr_t(SrcFd.Fd); + + ScopedFd DstFd(open(DstPath.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666)); + CHECK(bool(DstFd)); + void* DstNativeHandle = (void*)uintptr_t(DstFd.Fd); +# endif + + SUBCASE("CanClone returns true for same volume") { CHECK(CloneQuery->CanClone(SrcNativeHandle)); } + + SUBCASE("GetClonableRange and TryClone") + { + uint64_t PreBytes = 0; + uint64_t PostBytes = 0; + uint64_t Clonable = CloneQuery->GetClonableRange(0, 0, FileSize, PreBytes, PostBytes); + + if (Clonable > 0) + { + CHECK_EQ(PreBytes, 0); // Offset 0 is always aligned + CHECK(Clonable + PostBytes == FileSize); + + bool Cloned = CloneQuery->TryClone(SrcNativeHandle, DstNativeHandle, 0, 0, Clonable, FileSize); + CHECK(Cloned); + + if (Cloned) + { + // Write the post-alignment tail if any + if (PostBytes > 0) + { + const uint8_t* SrcData = SrcBuf.Data<uint8_t>() + Clonable; +# if ZEN_PLATFORM_WINDOWS + DWORD Written = 0; + OVERLAPPED Ov = {}; + Ov.Offset = (DWORD)(Clonable & 0xFFFFFFFF); + Ov.OffsetHigh = (DWORD)(Clonable >> 32); + ::WriteFile(DstHandle, SrcData, (DWORD)PostBytes, &Written, &Ov); +# else + pwrite(DstFd.Fd, SrcData, PostBytes, Clonable); +# endif + } + + // Close handles before reading back the file for verification +# if ZEN_PLATFORM_WINDOWS + SrcHandle.Close(); + DstHandle.Close(); +# else + SrcFd = ScopedFd(); + DstFd = ScopedFd(); +# endif + + FileContents DstContents = ReadFile(DstPath); + CHECK(DstContents); + IoBuffer DstFlat = DstContents.Flatten(); + CHECK_EQ(DstFlat.GetSize(), FileSize); + CHECK_EQ(memcmp(DstFlat.Data(), SrcBuf.Data(), FileSize), 0); + } + } + } + } + else + { + ZEN_INFO("CloneQueryInterface not available for {} (filesystem does not support block cloning)", TestBaseDir); + } + + DeleteDirectories(TestBaseDir); +} + TEST_CASE("SharedMemory") { CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index 16e2b59f8..6dc159a83 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -187,6 +187,14 @@ void ScanFile(void* NativeHandle, void WriteFile(void* NativeHandle, const void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); void ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uint64_t ChunkSize, std::error_code& Ec); +// Interface for sub-file range cloning on filesystems that support copy-on-write. +// GetCloneQueryInterface() returns nullptr on platforms without range clone support. +// +// Platform capabilities: +// Windows (ReFS) - True CoW range cloning via FSCTL_DUPLICATE_EXTENTS_TO_FILE. +// Linux (Btrfs/XFS) - True CoW range cloning via FICLONERANGE ioctl. +// macOS (APFS) - Not implemented. No sub-file range clone API exists. +// Whole-file CoW cloning is available via TryCloneFile (clonefile syscall). class CloneQueryInterface { public: diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 809312c7b..3177f64c1 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -6,6 +6,9 @@ #include <zencore/zencore.h> #include <filesystem> +#include <string> +#include <utility> +#include <vector> namespace zen { @@ -68,6 +71,12 @@ struct CreateProcOptions const std::filesystem::path* WorkingDirectory = nullptr; uint32_t Flags = 0; std::filesystem::path StdoutFile; + + /// Additional environment variables for the child process. These are merged + /// with the parent's environment — existing variables are inherited, and + /// entries here override or add to them. + std::vector<std::pair<std::string, std::string>> Environment; + #if ZEN_PLATFORM_WINDOWS JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed #endif diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 7b46f0e38..60293a313 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -333,7 +333,8 @@ public: #if defined(__clang__) && !defined(__APPLE__) && !defined(_MSC_VER) /* UE Toolchain Clang has different types for int64_t and long long so an override is - needed here. Without it, Clang can't disambiguate integer overloads */ + needed here. Without it, Clang can't disambiguate integer overloads. + On MSVC ABI (including clang-cl), int64_t is long long so no separate overload is needed. */ inline StringBuilderImpl& operator<<(long long n) { IntNum Str(n); diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index a67999e52..2e39cc660 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -17,6 +17,7 @@ std::string_view GetOperatingSystemName(); std::string GetOperatingSystemVersion(); std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); +std::string_view GetCompilerName(); struct SystemMetrics { diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index 013c51535..ccbea1282 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -193,8 +193,12 @@ private: std::atomic_uint32_t CallstackIdCounter{1}; // 0 is reserved for "unknown callstack" }; +} // namespace zen + # if UE_CALLSTACK_TRACE_USE_UNWIND_TABLES +namespace zen { + /* * Windows' x64 binaries contain a ".pdata" section that describes the location * and size of its functions and details on how to unwind them. The unwind @@ -908,10 +912,12 @@ FBacktracer::GetBacktraceId(void* AddressOfReturnAddress) // queue (i.e. the processing thread has caught up processing). return CallstackTracer.AddCallstack(BacktraceEntry); } -} +} // namespace zen # else // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES +namespace zen { + //////////////////////////////////////////////////////////////////////////////// class FBacktracer { @@ -921,8 +927,8 @@ public: static FBacktracer* Get(); inline uint32_t GetBacktraceId(void* AddressOfReturnAddress); uint32_t GetBacktraceId(uint64_t ReturnAddress); - void AddModule(uintptr_t Base, const char16_t* Name) {} - void RemoveModule(uintptr_t Base) {} + void AddModule(uintptr_t /*Base*/, const char16_t* /*Name*/) {} + void RemoveModule(uintptr_t /*Base*/) {} private: static FBacktracer* Instance; @@ -963,6 +969,7 @@ FBacktracer::GetBacktraceId(void* AddressOfReturnAddress) uint32_t FBacktracer::GetBacktraceId(uint64_t ReturnAddress) { + ZEN_UNUSED(ReturnAddress); # if !UE_BUILD_SHIPPING uint64_t StackFrames[256]; int32_t NumStackFrames = FPlatformStackWalk::CaptureStackBackTrace(StackFrames, UE_ARRAY_COUNT(StackFrames)); @@ -1006,7 +1013,8 @@ FBacktracer::GetBacktraceId(uint64_t ReturnAddress) return 0; } -} + +} // namespace zen # endif // UE_CALLSTACK_TRACE_USE_UNWIND_TABLES diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index f657869dc..080607f13 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -11,6 +11,7 @@ #include <zencore/timer.h> #include <zencore/trace.h> +#include <map> #include <thread> ZEN_THIRD_PARTY_INCLUDES_START @@ -20,8 +21,8 @@ ZEN_THIRD_PARTY_INCLUDES_START # include <Psapi.h> # include <shellapi.h> -# include <Shlobj.h> -# include <TlHelp32.h> +# include <shlobj.h> +# include <tlhelp32.h> #else # include <fcntl.h> # include <pthread.h> @@ -487,13 +488,57 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma STARTUPINFO StartupInfo{.cb = sizeof(STARTUPINFO)}; bool InheritHandles = false; - void* Environment = nullptr; LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; + // Build environment block when custom environment variables are specified + ExtendableWideStringBuilder<512> EnvironmentBlock; + void* Environment = nullptr; + if (!Options.Environment.empty()) + { + // Capture current environment into a map + std::map<std::wstring, std::wstring> EnvMap; + wchar_t* EnvStrings = GetEnvironmentStringsW(); + if (EnvStrings) + { + for (const wchar_t* Ptr = EnvStrings; *Ptr; Ptr += wcslen(Ptr) + 1) + { + std::wstring_view Entry(Ptr); + size_t EqPos = Entry.find(L'='); + if (EqPos != std::wstring_view::npos && EqPos > 0) + { + EnvMap[std::wstring(Entry.substr(0, EqPos))] = std::wstring(Entry.substr(EqPos + 1)); + } + } + FreeEnvironmentStringsW(EnvStrings); + } + + // Apply overrides + for (const auto& [Key, Value] : Options.Environment) + { + EnvMap[Utf8ToWide(Key)] = Utf8ToWide(Value); + } + + // Build double-null-terminated environment block + for (const auto& [Key, Value] : EnvMap) + { + EnvironmentBlock << Key; + EnvironmentBlock.Append(L'='); + EnvironmentBlock << Value; + EnvironmentBlock.Append(L'\0'); + } + EnvironmentBlock.Append(L'\0'); + + Environment = EnvironmentBlock.Data(); + } + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); DWORD CreationFlags = 0; + if (Environment) + { + CreationFlags |= CREATE_UNICODE_ENVIRONMENT; + } if (Options.Flags & CreateProcOptions::Flag_NewConsole) { CreationFlags |= CREATE_NEW_CONSOLE; @@ -790,6 +835,11 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine } } + for (const auto& [Key, Value] : Options.Environment) + { + setenv(Key.c_str(), Value.c_str(), 1); + } + if (execv(Executable.c_str(), ArgV.data()) < 0) { ThrowLastError("Failed to exec() a new process image"); diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 58b76783a..b7d01003b 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -31,6 +31,8 @@ namespace { struct SentryAssertImpl : zen::AssertImpl { + ZEN_DEBUG_SECTION ~SentryAssertImpl() override = default; + virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, int LineNumber, const char* FunctionName, diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 141450b84..8985a8a76 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -660,6 +660,24 @@ GetCpuName() #endif } +std::string_view +GetCompilerName() +{ +#define ZEN_STRINGIFY_IMPL(x) #x +#define ZEN_STRINGIFY(x) ZEN_STRINGIFY_IMPL(x) +#if ZEN_COMPILER_CLANG + return "clang " ZEN_STRINGIFY(__clang_major__) "." ZEN_STRINGIFY(__clang_minor__) "." ZEN_STRINGIFY(__clang_patchlevel__); +#elif ZEN_COMPILER_MSC + return "MSVC " ZEN_STRINGIFY(_MSC_VER); +#elif ZEN_COMPILER_GCC + return "GCC " ZEN_STRINGIFY(__GNUC__) "." ZEN_STRINGIFY(__GNUC_MINOR__) "." ZEN_STRINGIFY(__GNUC_PATCHLEVEL__); +#else + return "unknown"; +#endif +#undef ZEN_STRINGIFY +#undef ZEN_STRINGIFY_IMPL +} + void Describe(const SystemMetrics& Metrics, CbWriter& Writer) { diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index d7eb3b17d..f5bc723b1 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -24,6 +24,8 @@ # if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include <execinfo.h> # include <unistd.h> +# elif ZEN_PLATFORM_WINDOWS +# include <crtdbg.h> # endif namespace zen::testing { @@ -296,6 +298,17 @@ RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink } # endif +# if ZEN_PLATFORM_WINDOWS + // Suppress Windows error dialogs (crash/abort/assert) so tests terminate + // immediately instead of blocking on a modal dialog in CI or headless runs. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + _set_abort_behavior(0, _WRITE_ABORT_MSG); + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR); +# endif + zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); InstallCrashSignalHandlers(); diff --git a/src/zenhorde/hordetransportaes.cpp b/src/zenhorde/hordetransportaes.cpp index 986dd3705..505b6bde7 100644 --- a/src/zenhorde/hordetransportaes.cpp +++ b/src/zenhorde/hordetransportaes.cpp @@ -12,7 +12,7 @@ #if ZEN_PLATFORM_WINDOWS # include <zencore/windows.h> # include <bcrypt.h> -# pragma comment(lib, "Bcrypt.lib") +# pragma comment(lib, "bcrypt.lib") #else ZEN_THIRD_PARTY_INCLUDES_START # include <openssl/evp.h> diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp index 5f3ad2455..3ca586f87 100644 --- a/src/zenhttp/httpclient_test.cpp +++ b/src/zenhttp/httpclient_test.cpp @@ -154,6 +154,42 @@ public: }, HttpVerb::kGet); + m_Router.AddMatcher("anypath", [](std::string_view Str) -> bool { return !Str.empty(); }); + + m_Router.RegisterRoute( + "echo/uri", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string Body = std::string(HttpReq.RelativeUri()); + + auto Params = HttpReq.GetQueryParams(); + for (const auto& [Key, Value] : Params.KvPairs) + { + Body += fmt::format("\n{}={}", Key, Value); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body); + }, + HttpVerb::kGet | HttpVerb::kPut); + + m_Router.RegisterRoute( + "echo/uri/{anypath}", + [](HttpRouterRequest& Req) { + // Echo both the RelativeUri and the captured path segment + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view Captured = Req.GetCapture(1); + std::string Body = fmt::format("uri={}\ncapture={}", HttpReq.RelativeUri(), Captured); + + auto Params = HttpReq.GetQueryParams(); + for (const auto& [Key, Value] : Params.KvPairs) + { + Body += fmt::format("\n{}={}", Key, Value); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body); + }, + HttpVerb::kGet | HttpVerb::kPut); + m_Router.RegisterRoute( "slow", [](HttpRouterRequest& Req) { @@ -1689,6 +1725,77 @@ TEST_CASE("httpclient.https") # endif // ZEN_USE_OPENSSL +TEST_CASE("httpclient.uri_decoding") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + // URI without encoding — should pass through unchanged + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello/world.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/hello/world.txt\ncapture=hello/world.txt"); + } + + // Percent-encoded space — server should see decoded path + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt"); + } + + // Percent-encoded slash (%2F) — should be decoded to / + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/a%2Fb.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/a/b.txt\ncapture=a/b.txt"); + } + + // Multiple encodings in one path + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/file%20%26%20name.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/file & name.txt\ncapture=file & name.txt"); + } + + // No capture — echo/uri route returns just RelativeUri + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "echo/uri"); + } + + // Literal percent that is not an escape (%ZZ) — should be kept as-is + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/100%25done.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/100%done.txt\ncapture=100%done.txt"); + } + + // Query params — raw values are returned as-is from GetQueryParams + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri?key=value&name=test"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "echo/uri\nkey=value\nname=test"); + } + + // Query params with percent-encoded values + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri?prefix=listing%2F&mode=s3"); + REQUIRE(Resp.IsSuccess()); + // GetQueryParams returns raw (still-encoded) values — callers must Decode() explicitly + CHECK(Resp.AsText() == "echo/uri\nprefix=listing%2F\nmode=s3"); + } + + // Query params with path capture and encoding + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt?tag=a%26b"); + REQUIRE(Resp.IsSuccess()); + // Path is decoded, query values are raw + CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt\ntag=a%26b"); + } +} + TEST_SUITE_END(); void diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 69000dd8e..7bb02cb0f 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -700,15 +700,6 @@ HttpServerRequest::ReadPayloadPackage() ////////////////////////////////////////////////////////////////////////// void -HttpRequestRouter::AddPattern(const char* Id, const char* Regex) -{ - ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end()); - ZEN_ASSERT(!m_IsFinalized); - - m_PatternMap.insert({Id, Regex}); -} - -void HttpRequestRouter::AddMatcher(const char* Id, std::function<bool(std::string_view)>&& Matcher) { ZEN_ASSERT(m_MatcherNameMap.find(Id) == m_MatcherNameMap.end()); @@ -724,170 +715,77 @@ HttpRequestRouter::RegisterRoute(const char* UriPattern, HttpRequestRouter::Hand { ZEN_ASSERT(!m_IsFinalized); - if (ExtendableStringBuilder<128> ExpandedRegex; ProcessRegexSubstitutions(UriPattern, ExpandedRegex)) - { - // Regex route - m_RegexHandlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), UriPattern); - } - else - { - // New-style regex-free route. More efficient and should be used for everything eventually + int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern)); - int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern)); + int i = 0; - int i = 0; + std::vector<int> MatcherIndices; - std::vector<int> MatcherIndices; - - while (i < RegexLen) + while (i < RegexLen) + { + if (UriPattern[i] == '{') { - if (UriPattern[i] == '{') + bool IsComplete = false; + int PatternStart = i + 1; + while (++i < RegexLen) { - bool IsComplete = false; - int PatternStart = i + 1; - while (++i < RegexLen) + if (UriPattern[i] == '}') { - if (UriPattern[i] == '}') + if (i == PatternStart) { - if (i == PatternStart) - { - throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern)); - } - std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart); - if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end()) - { - // It's a match - MatcherIndices.push_back(it->second); - IsComplete = true; - ++i; - break; - } - else - { - throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern)); - } + throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern)); } - } - if (!IsComplete) - { - throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern)); - } - } - else - { - if (UriPattern[i] == '/') - { - throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern)); - } - - int SegmentStart = i; - while (++i < RegexLen && UriPattern[i] != '/') - ; - - std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart)); - int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size()); - m_Literals.push_back(std::string(Segment)); - MatcherIndices.push_back(-1 - LiteralIndex); - } - - if (i < RegexLen && UriPattern[i] == '/') - { - ++i; // skip slash - } - } - - m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern); - } -} - -std::string_view -HttpRouterRequest::GetCapture(uint32_t Index) const -{ - if (!m_CapturedSegments.empty()) - { - ZEN_ASSERT(Index < m_CapturedSegments.size()); - return m_CapturedSegments[Index]; - } - - ZEN_ASSERT(Index < m_Match.size()); - - const auto& Match = m_Match[Index]; - - return std::string_view(&*Match.first, Match.second - Match.first); -} - -bool -HttpRequestRouter::ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& OutExpandedRegex) -{ - size_t RegexLen = strlen(Regex); - - bool HasRegex = false; - - std::vector<std::string> UnknownPatterns; - - for (size_t i = 0; i < RegexLen;) - { - bool matched = false; - - if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\'))) - { - // Might have a pattern reference - find closing brace - - for (size_t j = i + 1; j < RegexLen; ++j) - { - if (Regex[j] == '}') - { - std::string Pattern(&Regex[i + 1], j - i - 1); - - if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end()) + std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart); + if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end()) { - OutExpandedRegex.Append(it->second.c_str()); - HasRegex = true; + // It's a match + MatcherIndices.push_back(it->second); + IsComplete = true; + ++i; + break; } else { - UnknownPatterns.push_back(Pattern); + throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern)); } - - // skip ahead - i = j + 1; - - matched = true; - - break; } } + if (!IsComplete) + { + throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern)); + } } - - if (!matched) - { - OutExpandedRegex.Append(Regex[i++]); - } - } - - if (HasRegex) - { - if (UnknownPatterns.size() > 0) + else { - std::string UnknownList; - for (const auto& Pattern : UnknownPatterns) + if (UriPattern[i] == '/') { - if (!UnknownList.empty()) - { - UnknownList += ", "; - } - UnknownList += "'"; - UnknownList += Pattern; - UnknownList += "'"; + throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern)); } - throw std::runtime_error(fmt::format("unknown pattern(s) {} in regex route '{}'", UnknownList, Regex)); + int SegmentStart = i; + while (++i < RegexLen && UriPattern[i] != '/') + ; + + std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart)); + int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size()); + m_Literals.push_back(std::string(Segment)); + MatcherIndices.push_back(-1 - LiteralIndex); } - return true; + if (i < RegexLen && UriPattern[i] == '/') + { + ++i; // skip slash + } } - return false; + m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern); +} + +std::string_view +HttpRouterRequest::GetCapture(uint32_t Index) const +{ + ZEN_ASSERT(Index < m_CapturedSegments.size()); + return m_CapturedSegments[Index]; } bool @@ -903,8 +801,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) std::string_view Uri = Request.RelativeUri(); HttpRouterRequest RouterRequest(Request); - // First try new-style matcher routes - for (const MatcherEndpoint& Handler : m_MatcherEndpoints) { if ((Handler.Verbs & Verb) == Verb) @@ -1002,28 +898,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) } } - // Old-style regex routes - - for (const auto& Handler : m_RegexHandlers) - { - if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx)) - { -#if ZEN_WITH_OTEL - if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan()) - { - ExtendableStringBuilder<128> RoutePath; - RoutePath.Append(Request.Service().BaseUri()); - RoutePath.Append(Handler.Pattern); - ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView()); - } -#endif - - Handler.Handler(RouterRequest); - - return true; // Route matched - } - } - return false; // No route matched } @@ -1423,72 +1297,6 @@ TEST_CASE("http.common") virtual uint32_t ParseRequestId() const override { return 0; } }; - SUBCASE("router-regex") - { - bool HandledA = false; - bool HandledAA = false; - std::vector<std::string> Captures; - auto Reset = [&] { - Captures.clear(); - HandledA = HandledAA = false; - }; - - TestHttpService Service; - - HttpRequestRouter r; - r.AddPattern("a", "([[:alpha:]]+)"); - r.RegisterRoute( - "{a}", - [&](auto& Req) { - HandledA = true; - Captures = {std::string(Req.GetCapture(0))}; - }, - HttpVerb::kGet); - - r.RegisterRoute( - "{a}/{a}", - [&](auto& Req) { - HandledAA = true; - Captures = {std::string(Req.GetCapture(1)), std::string(Req.GetCapture(2))}; - }, - HttpVerb::kGet); - - { - Reset(); - TestHttpServerRequest req(Service, "abc"sv); - r.HandleRequest(req); - CHECK(HandledA); - CHECK(!HandledAA); - REQUIRE_EQ(Captures.size(), 1); - CHECK_EQ(Captures[0], "abc"sv); - } - - { - Reset(); - TestHttpServerRequest req{Service, "abc/def"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - CHECK(HandledAA); - REQUIRE_EQ(Captures.size(), 2); - CHECK_EQ(Captures[0], "abc"sv); - CHECK_EQ(Captures[1], "def"sv); - } - - { - Reset(); - TestHttpServerRequest req{Service, "123"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - } - - { - Reset(); - TestHttpServerRequest req{Service, "a123"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - } - } - SUBCASE("router-matcher") { bool HandledA = false; diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 42e5b1628..c90e20aef 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -20,7 +20,6 @@ #include <gsl/gsl-lite.hpp> #include <list> #include <map> -#include <regex> #include <span> #include <unordered_map> @@ -358,9 +357,8 @@ class HttpRouterRequest public: /** Get captured segment from matched URL * - * @param Index Index of captured segment to retrieve. Note that due to - * backwards compatibility with regex-based routes, this index is 1-based - * and index=0 is the full matched URL + * @param Index Index of captured segment to retrieve. Index 0 is the full + * matched URL, subsequent indices are the matched segments in order. * @return Returns string view of captured segment */ std::string_view GetCapture(uint32_t Index) const; @@ -373,11 +371,8 @@ private: HttpRouterRequest(const HttpRouterRequest&) = delete; HttpRouterRequest& operator=(const HttpRouterRequest&) = delete; - using MatchResults_t = std::match_results<std::string_view::const_iterator>; - HttpServerRequest& m_HttpRequest; - MatchResults_t m_Match; - std::vector<std::string_view> m_CapturedSegments; // for matcher-based routes + std::vector<std::string_view> m_CapturedSegments; friend class HttpRequestRouter; }; @@ -385,9 +380,7 @@ private: /** HTTP request router helper * * This helper class allows a service implementer to register one or more - * endpoints using pattern matching. We currently support a legacy regex-based - * matching system, but also a new matcher-function based system which is more - * efficient and should be used whenever possible. + * endpoints using pattern matching with matcher functions. * * This is intended to be initialized once only, there is no thread * safety so you can absolutely not add or remove endpoints once the handler @@ -406,13 +399,6 @@ public: typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t; /** - * @brief Add pattern which can be referenced by name, commonly used for URL components - * @param Id String used to identify patterns for replacement - * @param Regex String which will replace the Id string in any registered URL paths - */ - void AddPattern(const char* Id, const char* Regex); - - /** * @brief Add matcher function which can be referenced by name, used for URL components * @param Id String used to identify matchers in endpoint specifications * @param Matcher Function which will be called to match the component @@ -422,8 +408,8 @@ public: /** * @brief Register an endpoint handler for the given route * @param Pattern Pattern used to match the handler to a request. This should - * only contain literal URI segments and pattern aliases registered - via AddPattern() or AddMatcher() + * only contain literal URI segments and matcher aliases registered + via AddMatcher() * @param HandlerFunc Handler function to call for any matching request * @param SupportedVerbs Supported HTTP verbs for this handler */ @@ -438,36 +424,6 @@ public: bool HandleRequest(zen::HttpServerRequest& Request); private: - bool ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& ExpandedRegex); - - struct RegexEndpoint - { - RegexEndpoint(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) - : RegEx(Regex, std::regex::icase | std::regex::ECMAScript) - , Verbs(SupportedVerbs) - , Handler(std::move(Handler)) - , Pattern(Pattern) - { - } - - ~RegexEndpoint() = default; - - std::regex RegEx; - HttpVerb Verbs; - HandlerFunc_t Handler; - const char* Pattern; - - private: - RegexEndpoint& operator=(const RegexEndpoint&) = delete; - RegexEndpoint(const RegexEndpoint&) = delete; - }; - - std::list<RegexEndpoint> m_RegexHandlers; - std::unordered_map<std::string, std::string> m_PatternMap; - - // New-style matcher endpoints. Should be preferred over regex endpoints where possible - // as it is considerably more efficient - struct MatcherEndpoint { MatcherEndpoint(std::vector<int>&& ComponentIndices, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 643f33618..9f4875eaf 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -601,6 +601,7 @@ public: bool m_IsLocalMachineRequest; bool m_AllowZeroCopyFileSend = true; std::string m_RemoteAddress; + std::string m_DecodedUri; // Percent-decoded URI; m_Uri/m_UriWithExtension point into this std::unique_ptr<HttpResponse> m_Response; }; @@ -623,6 +624,7 @@ public: ~HttpResponse() = default; void SetAllowZeroCopyFileSend(bool Allow) { m_AllowZeroCopyFileSend = Allow; } + void SetKeepAlive(bool KeepAlive) { m_IsKeepAlive = KeepAlive; } /** * Initialize the response for sending a payload made up of multiple blobs @@ -780,8 +782,8 @@ public: return m_Headers; } - template<typename SocketType> - void SendResponse(SocketType& Socket, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token) + template<typename SocketType, typename Executor> + void SendResponse(SocketType& Socket, Executor& Strand, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token) { ZEN_ASSERT(m_State == State::kInitialized); @@ -791,11 +793,11 @@ public: m_SendCb = std::move(Token); m_State = State::kSending; - SendNextChunk(Socket); + SendNextChunk(Socket, Strand); } - template<typename SocketType> - void SendNextChunk(SocketType& Socket) + template<typename SocketType, typename Executor> + void SendNextChunk(SocketType& Socket, Executor& Strand) { ZEN_ASSERT(m_State == State::kSending); @@ -812,12 +814,12 @@ public: auto CompletionToken = [Self = this, Token = std::move(m_SendCb), TotalBytes = m_TotalBytesSent] { Token({}, TotalBytes); }; - asio::defer(Socket.get_executor(), std::move(CompletionToken)); + asio::defer(Strand, std::move(CompletionToken)); return; } - auto OnCompletion = [this, &Socket](const asio::error_code& Ec, std::size_t ByteCount) { + auto OnCompletion = asio::bind_executor(Strand, [this, &Socket, &Strand](const asio::error_code& Ec, std::size_t ByteCount) { ZEN_ASSERT(m_State == State::kSending); m_TotalBytesSent += ByteCount; @@ -828,9 +830,9 @@ public: } else { - SendNextChunk(Socket); + SendNextChunk(Socket, Strand); } - }; + }); const IoVec& Io = m_IoVecs[m_IoVecCursor++]; @@ -982,16 +984,14 @@ private: void CloseConnection(); void SendInlineResponse(uint32_t RequestNumber, std::string_view StatusLine, std::string_view Headers = {}, std::string_view Body = {}); - HttpAsioServerImpl& m_Server; - asio::streambuf m_RequestBuffer; - std::atomic<uint32_t> m_RequestCounter{0}; - uint32_t m_ConnectionId = 0; - Ref<IHttpPackageHandler> m_PackageHandler; - - RwLock m_ActiveResponsesLock; + HttpAsioServerImpl& m_Server; + std::unique_ptr<SocketType> m_Socket; + asio::strand<asio::any_io_executor> m_Strand; + asio::streambuf m_RequestBuffer; + uint32_t m_RequestCounter = 0; + uint32_t m_ConnectionId = 0; + Ref<IHttpPackageHandler> m_PackageHandler; std::deque<std::unique_ptr<HttpResponse>> m_ActiveResponses; - - std::unique_ptr<SocketType> m_Socket; }; std::atomic<uint32_t> g_ConnectionIdCounter{0}; @@ -999,8 +999,9 @@ std::atomic<uint32_t> g_ConnectionIdCounter{0}; template<typename SocketType> HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Server, std::unique_ptr<SocketType>&& Socket) : m_Server(Server) -, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1)) , m_Socket(std::move(Socket)) +, m_Strand(asio::make_strand(m_Socket->get_executor())) +, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1)) { ZEN_TRACE_VERBOSE("new connection #{}", m_ConnectionId); } @@ -1008,8 +1009,6 @@ HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Ser template<typename SocketType> HttpServerConnectionT<SocketType>::~HttpServerConnectionT() { - RwLock::ExclusiveLockScope _(m_ActiveResponsesLock); - ZEN_TRACE_VERBOSE("destroying connection #{}", m_ConnectionId); } @@ -1017,7 +1016,7 @@ template<typename SocketType> void HttpServerConnectionT<SocketType>::HandleNewRequest() { - EnqueueRead(); + asio::dispatch(m_Strand, [Conn = AsSharedPtr()] { Conn->EnqueueRead(); }); } template<typename SocketType> @@ -1058,7 +1057,9 @@ HttpServerConnectionT<SocketType>::EnqueueRead() asio::async_read(*m_Socket.get(), m_RequestBuffer, asio::transfer_at_least(1), - [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnDataReceived(Ec, ByteCount); }); + asio::bind_executor(m_Strand, [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnDataReceived(Ec, ByteCount); + })); } template<typename SocketType> @@ -1091,7 +1092,7 @@ HttpServerConnectionT<SocketType>::OnDataReceived(const asio::error_code& Ec, [[ ZEN_TRACE_VERBOSE("on data received, connection: {}, request: {}, thread: {}, bytes: {}", m_ConnectionId, - m_RequestCounter.load(std::memory_order_relaxed), + m_RequestCounter, zen::GetCurrentThreadId(), NiceBytes(ByteCount)); @@ -1153,25 +1154,23 @@ HttpServerConnectionT<SocketType>::OnResponseDataSent(const asio::error_code& if (ResponseToPop) { - m_ActiveResponsesLock.WithExclusiveLock([&] { - // Once a response is sent we can release any referenced resources - // - // completion callbacks may be issued out-of-order so we need to - // remove the relevant entry from our active response list, it may - // not be the first - - if (auto It = find_if(begin(m_ActiveResponses), - end(m_ActiveResponses), - [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; }); - It != end(m_ActiveResponses)) - { - m_ActiveResponses.erase(It); - } - else - { - ZEN_WARN("response not found"); - } - }); + // Once a response is sent we can release any referenced resources + // + // completion callbacks may be issued out-of-order so we need to + // remove the relevant entry from our active response list, it may + // not be the first + + if (auto It = find_if(begin(m_ActiveResponses), + end(m_ActiveResponses), + [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; }); + It != end(m_ActiveResponses)) + { + m_ActiveResponses.erase(It); + } + else + { + ZEN_WARN("response not found"); + } } if (!m_RequestData.IsKeepAlive()) @@ -1234,9 +1233,11 @@ HttpServerConnectionT<SocketType>::SendInlineResponse(uint32_t RequestNumber asio::async_write( *m_Socket, Buffer, - [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + })); } template<typename SocketType> @@ -1272,21 +1273,23 @@ HttpServerConnectionT<SocketType>::HandleRequest() asio::async_write( *m_Socket, asio::buffer(ResponseStr->data(), ResponseStr->size()), - [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) { - if (Ec) - { - ZEN_WARN("WebSocket 101 send failed: {}", Ec.message()); - return; - } - - Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened(); - using WsConnType = WsAsioConnectionT<SocketType>; - Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer)); - Ref<WebSocketConnection> WsConnRef(WsConn.Get()); - - WsHandler->OnWebSocketOpen(std::move(WsConnRef)); - WsConn->Start(); - }); + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) { + if (Ec) + { + ZEN_WARN("WebSocket 101 send failed: {}", Ec.message()); + return; + } + + Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened(); + using WsConnType = WsAsioConnectionT<SocketType>; + Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer)); + Ref<WebSocketConnection> WsConnRef(WsConn.Get()); + + WsHandler->OnWebSocketOpen(std::move(WsConnRef)); + WsConn->Start(); + })); m_RequestState = RequestState::kDone; return; @@ -1312,7 +1315,7 @@ HttpServerConnectionT<SocketType>::HandleRequest() m_RequestState = RequestState::kWriting; } - const uint32_t RequestNumber = m_RequestCounter.fetch_add(1); + const uint32_t RequestNumber = m_RequestCounter++; if (HttpService* Service = m_Server.RouteRequest(m_RequestData.Url())) { @@ -1444,31 +1447,34 @@ HttpServerConnectionT<SocketType>::HandleRequest() { ZEN_TRACE_CPU("asio::async_write"); - std::string_view Headers = Response->GetHeaders(); + HttpResponse* ResponseRaw = Response.get(); + m_ActiveResponses.push_back(std::move(Response)); + + std::string_view Headers = ResponseRaw->GetHeaders(); std::vector<asio::const_buffer> AsioBuffers; AsioBuffers.push_back(asio::const_buffer(Headers.data(), Headers.size())); - asio::async_write(*m_Socket.get(), - AsioBuffers, - asio::transfer_all(), - [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + asio::async_write( + *m_Socket.get(), + AsioBuffers, + asio::transfer_all(), + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw); + })); } else { ZEN_TRACE_CPU("asio::async_write"); HttpResponse* ResponseRaw = Response.get(); - - m_ActiveResponsesLock.WithExclusiveLock([&] { - // Keep referenced resources alive - m_ActiveResponses.push_back(std::move(Response)); - }); + m_ActiveResponses.push_back(std::move(Response)); ResponseRaw->SendResponse( *m_Socket, + m_Strand, [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw); }); @@ -1982,11 +1988,24 @@ HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, { const int PrefixLength = Service.UriPrefixLength(); - std::string_view Uri = Request.Url(); - Uri.remove_prefix(std::min(PrefixLength, static_cast<int>(Uri.size()))); - m_Uri = Uri; - m_UriWithExtension = Uri; - m_QueryString = Request.QueryString(); + std::string_view RawUri = Request.Url(); + RawUri.remove_prefix(std::min(PrefixLength, static_cast<int>(RawUri.size()))); + + // Percent-decode the URI path so handlers see the same decoded paths regardless + // of whether the ASIO or http.sys backend is used (http.sys pre-decodes via CookedUrl). + // Skip the allocation when there is nothing to decode (common case). + if (RawUri.find('%') != std::string_view::npos) + { + m_DecodedUri = Decode(RawUri); + m_Uri = m_DecodedUri; + m_UriWithExtension = m_DecodedUri; + } + else + { + m_Uri = RawUri; + m_UriWithExtension = RawUri; + } + m_QueryString = Request.QueryString(); m_Verb = Request.RequestVerb(); m_ContentLength = Request.Body().Size(); @@ -2083,6 +2102,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode) m_Response.reset(new HttpResponse(HttpContentType::kBinary, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); std::array<IoBuffer, 0> Empty; m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty); @@ -2097,6 +2117,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs); } @@ -2108,6 +2129,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_ASSERT(!m_Response); m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); IoBuffer MessageBuffer(IoBuffer::Wrap, ResponseString.data(), ResponseString.size()); std::array<IoBuffer, 1> SingleBufferList({MessageBuffer}); diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index a1bb719c8..31b0315d4 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -147,7 +147,7 @@ public: HttpPluginServerRequest& operator=(const HttpPluginServerRequest&) = delete; // As this is plugin transport connection used for specialized connections we assume it is not a machine local connection - bool IsLocalMachineRequest() const override { return false; } + virtual bool IsLocalMachineRequest() const override { return false; } virtual std::string_view GetAuthorizationHeader() const override; virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index eaf080960..2074fbd79 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -1176,7 +1176,7 @@ HttpSysServer::RegisterHttpUrls(int BasePort) { Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, WildcardUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); - if ((Result == ERROR_SHARING_VIOLATION)) + if (Result == ERROR_SHARING_VIOLATION) { ZEN_INFO("Desired port {} is in use (HttpAddUrlToUrlGroup returned: {}), retrying", EffectivePort, diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index a02ca7be3..0d8550c5b 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -865,7 +865,7 @@ ZenComputeServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as COMPUTE (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index 696991403..b0ae0a8b1 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -337,7 +337,7 @@ ZenHubServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as HUB (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp index acfdad45f..c768e940a 100644 --- a/src/zenserver/proxy/zenproxyserver.cpp +++ b/src/zenserver/proxy/zenproxyserver.cpp @@ -359,7 +359,7 @@ ZenProxyServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as PROXY (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp index 052c3d630..e347e2dfe 100644 --- a/src/zenserver/storage/objectstore/objectstore.cpp +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -271,7 +271,7 @@ HttpObjectStoreService::Inititalize() CreateDirectories(BucketsPath); } - static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]() ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; m_Router.AddMatcher("path", @@ -292,10 +292,9 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "bucket/{path}", [this](zen::HttpRouterRequest& Request) { - const std::string_view EncodedPath = Request.GetCapture(1); - const std::string Path = Request.ServerRequest().Decode(EncodedPath); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0; if (IsObject) { @@ -378,7 +377,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s const auto QueryParms = Request.ServerRequest().GetQueryParams(); if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) { - BucketPrefix = PrefixParam; + BucketPrefix = HttpServerRequest::Decode(PrefixParam); } } BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); diff --git a/src/zenserver/storage/vfs/vfsservice.cpp b/src/zenserver/storage/vfs/vfsservice.cpp index 863ec348a..f418c4131 100644 --- a/src/zenserver/storage/vfs/vfsservice.cpp +++ b/src/zenserver/storage/vfs/vfsservice.cpp @@ -62,7 +62,7 @@ GetContentAsCbObject(HttpServerRequest& HttpReq, CbObject& Cb) // echo {"method": "mount", "params": {"path": "d:\\VFS_ROOT"}} | curl.exe http://localhost:8558/vfs --data-binary @- // echo {"method": "unmount"} | curl.exe http://localhost:8558/vfs --data-binary @- -VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_StatusService(StatusService), m_Impl(ServiceImpl) +VfsService::VfsService(HttpStatusService& StatusService, VfsServiceImpl* ServiceImpl) : m_Impl(ServiceImpl), m_StatusService(StatusService) { m_Router.RegisterRoute( "info", diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index 77588bd6c..bba5e0a61 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -720,7 +720,7 @@ ZenStorageServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/targetver.h b/src/zenserver/targetver.h index d432d6993..4805141de 100644 --- a/src/zenserver/targetver.h +++ b/src/zenserver/targetver.h @@ -7,4 +7,4 @@ // If you wish to build your application for a previous Windows platform, include WinSDKVer.h and // set the _WIN32_WINNT macro to the platform you wish to support before including SDKDDKVer.h. -#include <SDKDDKVer.h> +#include <sdkddkver.h> diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 52889fff5..6b29dadfb 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -60,13 +60,15 @@ target("zenserver") end if is_plat("windows") then - add_ldflags("/subsystem:console,5.02") - add_ldflags("/MANIFEST:EMBED") - add_ldflags("/LTCG") + add_ldflags("/subsystem:console,5.02", {force = true}) + add_ldflags("/MANIFEST:EMBED", {force = true}) + if not (get_config("toolchain") or ""):find("clang") then + add_ldflags("/LTCG") + end add_files("zenserver.rc") add_cxxflags("/bigobj") add_links("delayimp", "projectedfslib") - add_ldflags("/delayload:ProjectedFSLib.dll") + add_ldflags("/delayload:ProjectedFSLib.dll", {force = true}) else remove_files("windows/**") end diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 519176ffe..6760e0372 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -201,6 +201,9 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: std::chrono::system_clock::now().time_since_epoch()).count(), .BuildOptions = { {"ZEN_ADDRESS_SANITIZER", ZEN_ADDRESS_SANITIZER != 0}, + {"ZEN_THREAD_SANITIZER", ZEN_THREAD_SANITIZER != 0}, + {"ZEN_MEMORY_SANITIZER", ZEN_MEMORY_SANITIZER != 0}, + {"ZEN_LEAK_SANITIZER", ZEN_LEAK_SANITIZER != 0}, {"ZEN_USE_SENTRY", ZEN_USE_SENTRY != 0}, {"ZEN_WITH_TESTS", ZEN_WITH_TESTS != 0}, {"ZEN_USE_MIMALLOC", ZEN_USE_MIMALLOC != 0}, @@ -251,6 +254,12 @@ ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) co OutOptions << "ZEN_ADDRESS_SANITIZER=" << (ZEN_ADDRESS_SANITIZER ? "1" : "0"); OutOptions << Separator; + OutOptions << "ZEN_THREAD_SANITIZER=" << (ZEN_THREAD_SANITIZER ? "1" : "0"); + OutOptions << Separator; + OutOptions << "ZEN_MEMORY_SANITIZER=" << (ZEN_MEMORY_SANITIZER ? "1" : "0"); + OutOptions << Separator; + OutOptions << "ZEN_LEAK_SANITIZER=" << (ZEN_LEAK_SANITIZER ? "1" : "0"); + OutOptions << Separator; OutOptions << "ZEN_USE_SENTRY=" << (ZEN_USE_SENTRY ? "1" : "0"); OutOptions << Separator; OutOptions << "ZEN_WITH_TESTS=" << (ZEN_WITH_TESTS ? "1" : "0"); diff --git a/src/zenserver/zenserver.rc b/src/zenserver/zenserver.rc index f353bd9cc..abe1acf71 100644 --- a/src/zenserver/zenserver.rc +++ b/src/zenserver/zenserver.rc @@ -28,7 +28,7 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US // Icon with lowest ID value placed first to ensure application icon // remains consistent on all systems. -IDI_ICON1 ICON "..\\zen.ico" +IDI_ICON1 ICON "../zen.ico" #endif // English (United States) resources ///////////////////////////////////////////////////////////////////////////// diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index b3450b805..f3edf804d 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1776,11 +1776,13 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64())); m_LastGcExpireTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count()))); - if (m_LastGcTime + m_Config.Interval < GcClock::Now()) + if (m_LastGcTime > GcClock::Now() || m_LastGcTime + m_Config.Interval < GcClock::Now()) { - // TODO: Trigger GC? + // Reset if the stored timestamp is in the future (e.g. clock resolution mismatch + // between the build that wrote gc_state and this build) or too far in the past. m_LastGcTime = GcClock::Now(); m_LastLightweightGcTime = m_LastGcTime; + m_LastGcExpireTime = GcClock::TimePoint::min(); } m_AttachmentPassIndex = SchedulerState["AttachmentPassIndex"sv].AsUInt8(); } @@ -2084,6 +2086,10 @@ GcScheduler::GetState() const { Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero(); } + else if (Result.RemainingTimeUntilFullGc > Result.Config.Interval) + { + Result.RemainingTimeUntilFullGc = Result.Config.Interval; + } Result.RemainingTimeUntilLightweightGc = Result.Config.LightweightInterval.count() == 0 @@ -2094,6 +2100,10 @@ GcScheduler::GetState() const { Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero(); } + else if (Result.RemainingTimeUntilLightweightGc > Result.Config.LightweightInterval) + { + Result.RemainingTimeUntilLightweightGc = Result.Config.LightweightInterval; + } } return Result; @@ -2418,6 +2428,10 @@ GcScheduler::SchedulerThread() { RemainingTimeUntilGc = std::chrono::seconds::zero(); } + else if (RemainingTimeUntilGc > GcInterval) + { + RemainingTimeUntilGc = GcInterval; + } std::chrono::seconds RemainingTimeUntilLightweightGc = LightweightGcInterval.count() == 0 ? std::chrono::seconds::max() @@ -2428,6 +2442,10 @@ GcScheduler::SchedulerThread() { RemainingTimeUntilLightweightGc = std::chrono::seconds::zero(); } + else if (RemainingTimeUntilLightweightGc > LightweightGcInterval) + { + RemainingTimeUntilLightweightGc = LightweightGcInterval; + } // Don't schedule a lightweight GC if a full GC is // due quite soon anyway |