aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStefan Boberg <[email protected]>2026-03-16 10:27:24 +0100
committerGitHub Enterprise <[email protected]>2026-03-16 10:27:24 +0100
commit6df7bce35e84f91c868face688587c26a3765c7e (patch)
treea2bf78a9b708707a6b2484c0a00c70abbc2f1891
parentAdd Docker image build for compute workers (#837) (diff)
downloadzen-6df7bce35e84f91c868face688587c26a3765c7e.tar.xz
zen-6df7bce35e84f91c868face688587c26a3765c7e.zip
URI decoding, process env, compiler info, httpasio strands, regex route removal (#841)
- Percent-decode URIs in ASIO HTTP server to match http.sys CookedUrl behavior, ensuring consistent decoded paths across backends - Add Environment field to CreateProcOptions for passing extra env vars to child processes (Windows: merged into Unicode environment block; Unix: setenv in fork) - Add GetCompilerName() and include it in build options startup logging - Suppress Windows CRT error dialogs in test harness for headless/CI runs - Fix mimalloc package: pass CMAKE_BUILD_TYPE, skip cfuncs test for cross-compile - Add virtual destructor to SentryAssertImpl to fix debug-mode warning - Simplify object store path handling now that URIs arrive pre-decoded - Add URI decoding test coverage for percent-encoded paths and query params - Simplify httpasio request handling by using strands (guarantees no parallel handlers per connection) - Removed deprecated regex-based route matching support - Fix full GC never triggering after cross-toolchain builds: The `gc_state` file stores `system_clock` ticks, but the tick resolution differs between toolchains (nanoseconds on GCC/standard clang, microseconds on UE clang). A nanosecond timestamp misinterpreted as microseconds appears far in the future (~year 58,000), bypassing the staleness check and preventing time-based full GC from ever running. Fixed by also resetting when the stored timestamp is in the future. - Clamp GC countdown display to configured interval: Prevents nonsensical log output (e.g. "Full GC in 492128002h") caused by the above or any other clock anomaly. The clamp applies to both the scheduler log and the status API.
-rw-r--r--repo/packages/m/mimalloc/xmake.lua6
-rw-r--r--scripts/test.lua401
-rw-r--r--src/zencore/include/zencore/process.h9
-rw-r--r--src/zencore/include/zencore/system.h1
-rw-r--r--src/zencore/process.cpp52
-rw-r--r--src/zencore/sentryintegration.cpp2
-rw-r--r--src/zencore/system.cpp18
-rw-r--r--src/zencore/testing.cpp13
-rw-r--r--src/zenhttp/httpclient_test.cpp107
-rw-r--r--src/zenhttp/httpserver.cpp286
-rw-r--r--src/zenhttp/include/zenhttp/httpserver.h56
-rw-r--r--src/zenhttp/servers/httpasio.cpp180
-rw-r--r--src/zenserver/compute/computeserver.cpp2
-rw-r--r--src/zenserver/hub/zenhubserver.cpp2
-rw-r--r--src/zenserver/proxy/zenproxyserver.cpp2
-rw-r--r--src/zenserver/storage/objectstore/objectstore.cpp11
-rw-r--r--src/zenserver/storage/zenstorageserver.cpp2
-rw-r--r--src/zenserver/zenserver.cpp9
-rw-r--r--src/zenstore/gc.cpp22
-rw-r--r--tsan.supp12
-rw-r--r--xmake.lua418
21 files changed, 829 insertions, 782 deletions
diff --git a/repo/packages/m/mimalloc/xmake.lua b/repo/packages/m/mimalloc/xmake.lua
index 993e4c1a9..54d6613b8 100644
--- a/repo/packages/m/mimalloc/xmake.lua
+++ b/repo/packages/m/mimalloc/xmake.lua
@@ -35,7 +35,7 @@ package("mimalloc")
end
on_install("macosx", "windows", "linux", "android", "mingw", function (package)
- local configs = {"-DMI_OVERRIDE=OFF"}
+ local configs = {"-DMI_OVERRIDE=OFF", "-DCMAKE_BUILD_TYPE=" .. (package:is_debug() and "Debug" or "Release")}
table.insert(configs, "-DMI_BUILD_STATIC=" .. (package:config("shared") and "OFF" or "ON"))
table.insert(configs, "-DMI_BUILD_SHARED=" .. (package:config("shared") and "ON" or "OFF"))
table.insert(configs, "-DMI_SECURE=" .. (package:config("secure") and "ON" or "OFF"))
@@ -71,5 +71,7 @@ package("mimalloc")
end)
on_test(function (package)
- assert(package:has_cfuncs("mi_malloc", {includes = "mimalloc.h"}))
+ if not package:is_cross() then
+ assert(package:has_cfuncs("mi_malloc", {includes = "mimalloc.h"}))
+ end
end)
diff --git a/scripts/test.lua b/scripts/test.lua
new file mode 100644
index 000000000..df1218ce8
--- /dev/null
+++ b/scripts/test.lua
@@ -0,0 +1,401 @@
+-- Copyright Epic Games, Inc. All Rights Reserved.
+
+function main()
+ import("core.base.option")
+ import("core.project.config")
+ import("core.project.project")
+
+ config.load()
+
+ -- Override table: target name -> short name (for targets that don't follow convention)
+ local short_name_overrides = {
+ ["zenserver-test"] = "integration",
+ }
+
+ -- Build test list from targets in the "tests" group
+ local available_tests = {}
+ for name, target in pairs(project.targets()) do
+ if target:get("group") == "tests" and name:endswith("-test") then
+ local short = short_name_overrides[name]
+ if not short then
+ -- Derive short name: "zencore-test" -> "core"
+ short = name
+ if short:startswith("zen") then short = short:sub(4) end
+ if short:endswith("-test") then short = short:sub(1, -6) end
+ end
+ table.insert(available_tests, {short, name})
+ end
+ end
+
+ -- Add non-test-group entries that have a test subcommand
+ table.insert(available_tests, {"server", "zenserver"})
+
+ table.sort(available_tests, function(a, b) return a[1] < b[1] end)
+
+ -- Handle --list: print discovered test names and exit
+ if option.get("list") then
+ printf("Available tests:\n")
+ for _, entry in ipairs(available_tests) do
+ printf(" %-16s -> %s\n", entry[1], entry[2])
+ end
+ return
+ end
+
+ local testname = option.get("run")
+
+ -- Parse comma-separated test names into a set
+ local requested = {}
+ for token in testname:gmatch("[^,]+") do
+ requested[token:match("^%s*(.-)%s*$")] = true
+ end
+
+ -- Filter to requested test(s)
+ local tests = {}
+ local matched = {}
+
+ for _, entry in ipairs(available_tests) do
+ local name, target = entry[1], entry[2]
+ if requested["all"] or requested[name] then
+ table.insert(tests, {name = name, target = target})
+ matched[name] = true
+ end
+ end
+
+ -- Check for unknown test names
+ if not requested["all"] then
+ for name, _ in pairs(requested) do
+ if not matched[name] then
+ raise("no tests match specification: '%s'", name)
+ end
+ end
+ end
+
+ if #tests == 0 then
+ raise("no tests match specification: '%s'", testname)
+ end
+
+ local plat, arch
+ if is_host("windows") then
+ plat = "windows"
+ arch = "x64"
+ elseif is_host("macosx") then
+ plat = "macosx"
+ arch = is_arch("arm64") and "arm64" or "x86_64"
+ else
+ plat = "linux"
+ arch = "x86_64"
+ end
+
+ -- Only reconfigure if current config doesn't already match
+ if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then
+ local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or ""
+ local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or ""
+ os.exec("xmake config -y -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag)
+ end
+
+ -- Build targets we're going to run
+ if requested["all"] then
+ os.exec("xmake build -y")
+ else
+ for _, entry in ipairs(tests) do
+ os.exec("xmake build -y %s", entry.target)
+ end
+ end
+
+ local use_junit_reporting = option.get("junit")
+ local use_noskip = option.get("noskip")
+ local use_verbose = option.get("verbose")
+ local repeat_count = tonumber(option.get("repeat")) or 1
+ local extra_args = option.get("arguments") or {}
+ local junit_report_files = {}
+
+ local junit_report_dir
+ if use_junit_reporting then
+ junit_report_dir = path.join(os.projectdir(), config.get("buildir"), "reports")
+ os.mkdir(junit_report_dir)
+ end
+
+ -- Results collection for summary table
+ local results = {}
+ local any_failed = false
+
+ -- Format a number with thousands separators (e.g. 31103 -> "31,103")
+ local function format_number(n)
+ local s = tostring(n)
+ local pos = #s % 3
+ if pos == 0 then pos = 3 end
+ local result = s:sub(1, pos)
+ for i = pos + 1, #s, 3 do
+ result = result .. "," .. s:sub(i, i + 2)
+ end
+ return result
+ end
+
+ -- Center a string within a given width
+ local function center_str(s, width)
+ local pad = width - #s
+ local lpad = math.floor(pad / 2)
+ local rpad = pad - lpad
+ return string.rep(" ", lpad) .. s .. string.rep(" ", rpad)
+ end
+
+ -- Left-align a string within a given width (with 1-space left margin)
+ local function left_align_str(s, width)
+ return " " .. s .. string.rep(" ", width - #s - 1)
+ end
+
+ -- Right-align a string within a given width (with 1-space right margin)
+ local function right_align_str(s, width)
+ return string.rep(" ", width - #s - 1) .. s .. " "
+ end
+
+ -- Format elapsed seconds as a human-readable string
+ local function format_time(seconds)
+ if seconds >= 60 then
+ local mins = math.floor(seconds / 60)
+ local secs = seconds - mins * 60
+ return string.format("%dm %04.1fs", mins, secs)
+ else
+ return string.format("%.1fs", seconds)
+ end
+ end
+
+ -- Parse test summary file written by TestListener
+ local function parse_summary_file(filepath)
+ if not os.isfile(filepath) then return nil end
+ local content = io.readfile(filepath)
+ if not content then return nil end
+ local ct = content:match("cases_total=(%d+)")
+ local cp = content:match("cases_passed=(%d+)")
+ local at = content:match("assertions_total=(%d+)")
+ local ap = content:match("assertions_passed=(%d+)")
+ if ct then
+ local failures = {}
+ for name, file, line in content:gmatch("failed=([^|\n]+)|([^|\n]+)|(%d+)") do
+ table.insert(failures, {name = name, file = file, line = tonumber(line)})
+ end
+ local es = content:match("elapsed_seconds=([%d%.]+)")
+ return {
+ cases_total = tonumber(ct),
+ cases_passed = tonumber(cp) or 0,
+ asserts_total = tonumber(at) or 0,
+ asserts_passed = tonumber(ap) or 0,
+ elapsed_seconds = tonumber(es) or 0,
+ failures = failures
+ }
+ end
+ return nil
+ end
+
+ -- Temp directory for summary files
+ local summary_dir = path.join(os.tmpdir(), "zen-test-summary")
+ os.mkdir(summary_dir)
+
+ -- Run each test suite and collect results
+ for iteration = 1, repeat_count do
+ if repeat_count > 1 then
+ printf("\n*** Iteration %d/%d ***\n", iteration, repeat_count)
+ end
+
+ for _, entry in ipairs(tests) do
+ local name, target = entry.name, entry.target
+ printf("=== %s ===\n", target)
+
+ local suite_name = target
+ if name == "server" then
+ suite_name = "zenserver (test)"
+ end
+
+ local cmd = string.format("xmake run %s", target)
+ if name == "server" then
+ cmd = string.format("xmake run %s test", target)
+ end
+ cmd = string.format("%s --duration=true", cmd)
+
+ if use_junit_reporting then
+ local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target))
+ junit_report_files[target] = junit_report_file
+ cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file)
+ end
+ if use_noskip then
+ cmd = string.format("%s --no-skip", cmd)
+ end
+ if use_verbose and name == "integration" then
+ cmd = string.format("%s --verbose", cmd)
+ end
+ for _, arg in ipairs(extra_args) do
+ cmd = string.format("%s %s", cmd, arg)
+ end
+
+ -- Tell TestListener where to write the summary
+ local summary_file = path.join(summary_dir, target .. ".txt")
+ os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file)
+
+ -- Run test with real-time streaming output
+ local test_ok = true
+ try {
+ function()
+ os.exec(cmd)
+ end,
+ catch {
+ function(errors)
+ test_ok = false
+ end
+ }
+ }
+
+ -- Read summary written by TestListener
+ local summary = parse_summary_file(summary_file)
+ os.tryrm(summary_file)
+
+ if not test_ok then
+ any_failed = true
+ end
+
+ table.insert(results, {
+ suite = suite_name,
+ cases_passed = summary and summary.cases_passed or 0,
+ cases_total = summary and summary.cases_total or 0,
+ asserts_passed = summary and summary.asserts_passed or 0,
+ asserts_total = summary and summary.asserts_total or 0,
+ elapsed_seconds = summary and summary.elapsed_seconds or 0,
+ failures = summary and summary.failures or {},
+ passed = test_ok
+ })
+ end
+
+ if any_failed then
+ if repeat_count > 1 then
+ printf("\n*** Failure detected on iteration %d, stopping ***\n", iteration)
+ end
+ break
+ end
+ end
+
+ -- Clean up
+ os.setenv("ZEN_TEST_SUMMARY_FILE", "")
+ os.tryrm(summary_dir)
+
+ -- Print JUnit reports if requested
+ for test, junit_report_file in pairs(junit_report_files) do
+ printf("=== report - %s ===\n", test)
+ if os.isfile(junit_report_file) then
+ local data = io.readfile(junit_report_file)
+ if data then
+ print(data)
+ end
+ end
+ end
+
+ -- Print summary table
+ if #results > 0 then
+ -- Calculate column widths based on content
+ local col_suite = #("Suite")
+ local col_cases = #("Cases")
+ local col_asserts = #("Assertions")
+ local col_time = #("Time")
+ local col_status = #("Status")
+
+ -- Compute totals
+ local total_cases_passed = 0
+ local total_cases_total = 0
+ local total_asserts_passed = 0
+ local total_asserts_total = 0
+ local total_elapsed = 0
+
+ for _, r in ipairs(results) do
+ col_suite = math.max(col_suite, #r.suite)
+ local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total)
+ col_cases = math.max(col_cases, #cases_str)
+ local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total)
+ col_asserts = math.max(col_asserts, #asserts_str)
+ col_time = math.max(col_time, #format_time(r.elapsed_seconds))
+ local status_str = r.passed and "SUCCESS" or "FAILED"
+ col_status = math.max(col_status, #status_str)
+
+ total_cases_passed = total_cases_passed + r.cases_passed
+ total_cases_total = total_cases_total + r.cases_total
+ total_asserts_passed = total_asserts_passed + r.asserts_passed
+ total_asserts_total = total_asserts_total + r.asserts_total
+ total_elapsed = total_elapsed + r.elapsed_seconds
+ end
+
+ -- Account for totals row in column widths
+ col_suite = math.max(col_suite, #("Total"))
+ col_cases = math.max(col_cases, #(format_number(total_cases_passed) .. "/" .. format_number(total_cases_total)))
+ col_asserts = math.max(col_asserts, #(format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total)))
+ col_time = math.max(col_time, #format_time(total_elapsed))
+
+ -- Add padding (1 space each side)
+ col_suite = col_suite + 2
+ col_cases = col_cases + 2
+ col_asserts = col_asserts + 2
+ col_time = col_time + 2
+ col_status = col_status + 2
+
+ -- Build horizontal border segments
+ local h_suite = string.rep("-", col_suite)
+ local h_cases = string.rep("-", col_cases)
+ local h_asserts = string.rep("-", col_asserts)
+ local h_time = string.rep("-", col_time)
+ local h_status = string.rep("-", col_status)
+
+ local top = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
+ local mid = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
+ local bottom = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
+ local vbar = "|"
+
+ local header_msg = any_failed and "Some tests failed:" or "All tests passed:"
+ printf("\n* %s\n", header_msg)
+ printf(" %s\n", top)
+ printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, center_str("Suite", col_suite), vbar, center_str("Cases", col_cases), vbar, center_str("Assertions", col_asserts), vbar, center_str("Time", col_time), vbar, center_str("Status", col_status), vbar)
+
+ for _, r in ipairs(results) do
+ printf(" %s\n", mid)
+ local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total)
+ local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total)
+ local time_str = format_time(r.elapsed_seconds)
+ local status_str = r.passed and "SUCCESS" or "FAILED"
+ printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str(r.suite, col_suite), vbar, right_align_str(cases_str, col_cases), vbar, right_align_str(asserts_str, col_asserts), vbar, right_align_str(time_str, col_time), vbar, right_align_str(status_str, col_status), vbar)
+ end
+
+ -- Totals row
+ if #results > 1 then
+ local h_suite_eq = string.rep("=", col_suite)
+ local h_cases_eq = string.rep("=", col_cases)
+ local h_asserts_eq = string.rep("=", col_asserts)
+ local h_time_eq = string.rep("=", col_time)
+ local h_status_eq = string.rep("=", col_status)
+ local totals_sep = "+" .. h_suite_eq .. "+" .. h_cases_eq .. "+" .. h_asserts_eq .. "+" .. h_time_eq .. "+" .. h_status_eq .. "+"
+ printf(" %s\n", totals_sep)
+
+ local total_cases_str = format_number(total_cases_passed) .. "/" .. format_number(total_cases_total)
+ local total_asserts_str = format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total)
+ local total_time_str = format_time(total_elapsed)
+ local total_status_str = any_failed and "FAILED" or "SUCCESS"
+ printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str("Total", col_suite), vbar, right_align_str(total_cases_str, col_cases), vbar, right_align_str(total_asserts_str, col_asserts), vbar, right_align_str(total_time_str, col_time), vbar, right_align_str(total_status_str, col_status), vbar)
+ end
+
+ printf(" %s\n", bottom)
+ end
+
+ -- Print list of individual failing tests
+ if any_failed then
+ printf("\n Failures:\n")
+ for _, r in ipairs(results) do
+ if #r.failures > 0 then
+ printf(" -- %s --\n", r.suite)
+ for _, f in ipairs(r.failures) do
+ printf(" FAILED: %s (%s:%d)\n", f.name, f.file, f.line)
+ end
+ elseif not r.passed then
+ printf(" -- %s --\n", r.suite)
+ printf(" (test binary exited with error, no failure details available)\n")
+ end
+ end
+ end
+
+ if any_failed then
+ raise("one or more test suites failed")
+ end
+end
diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h
index 809312c7b..3177f64c1 100644
--- a/src/zencore/include/zencore/process.h
+++ b/src/zencore/include/zencore/process.h
@@ -6,6 +6,9 @@
#include <zencore/zencore.h>
#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
namespace zen {
@@ -68,6 +71,12 @@ struct CreateProcOptions
const std::filesystem::path* WorkingDirectory = nullptr;
uint32_t Flags = 0;
std::filesystem::path StdoutFile;
+
+ /// Additional environment variables for the child process. These are merged
+ /// with the parent's environment — existing variables are inherited, and
+ /// entries here override or add to them.
+ std::vector<std::pair<std::string, std::string>> Environment;
+
#if ZEN_PLATFORM_WINDOWS
JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed
#endif
diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h
index a67999e52..2e39cc660 100644
--- a/src/zencore/include/zencore/system.h
+++ b/src/zencore/include/zencore/system.h
@@ -17,6 +17,7 @@ std::string_view GetOperatingSystemName();
std::string GetOperatingSystemVersion();
std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos"
std::string_view GetCpuName();
+std::string_view GetCompilerName();
struct SystemMetrics
{
diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp
index f657869dc..852678ffe 100644
--- a/src/zencore/process.cpp
+++ b/src/zencore/process.cpp
@@ -11,6 +11,7 @@
#include <zencore/timer.h>
#include <zencore/trace.h>
+#include <map>
#include <thread>
ZEN_THIRD_PARTY_INCLUDES_START
@@ -487,13 +488,57 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma
STARTUPINFO StartupInfo{.cb = sizeof(STARTUPINFO)};
bool InheritHandles = false;
- void* Environment = nullptr;
LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr;
LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr;
+ // Build environment block when custom environment variables are specified
+ ExtendableWideStringBuilder<512> EnvironmentBlock;
+ void* Environment = nullptr;
+ if (!Options.Environment.empty())
+ {
+ // Capture current environment into a map
+ std::map<std::wstring, std::wstring> EnvMap;
+ wchar_t* EnvStrings = GetEnvironmentStringsW();
+ if (EnvStrings)
+ {
+ for (const wchar_t* Ptr = EnvStrings; *Ptr; Ptr += wcslen(Ptr) + 1)
+ {
+ std::wstring_view Entry(Ptr);
+ size_t EqPos = Entry.find(L'=');
+ if (EqPos != std::wstring_view::npos && EqPos > 0)
+ {
+ EnvMap[std::wstring(Entry.substr(0, EqPos))] = std::wstring(Entry.substr(EqPos + 1));
+ }
+ }
+ FreeEnvironmentStringsW(EnvStrings);
+ }
+
+ // Apply overrides
+ for (const auto& [Key, Value] : Options.Environment)
+ {
+ EnvMap[Utf8ToWide(Key)] = Utf8ToWide(Value);
+ }
+
+ // Build double-null-terminated environment block
+ for (const auto& [Key, Value] : EnvMap)
+ {
+ EnvironmentBlock << Key;
+ EnvironmentBlock.Append(L'=');
+ EnvironmentBlock << Value;
+ EnvironmentBlock.Append(L'\0');
+ }
+ EnvironmentBlock.Append(L'\0');
+
+ Environment = EnvironmentBlock.Data();
+ }
+
const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid();
DWORD CreationFlags = 0;
+ if (Environment)
+ {
+ CreationFlags |= CREATE_UNICODE_ENVIRONMENT;
+ }
if (Options.Flags & CreateProcOptions::Flag_NewConsole)
{
CreationFlags |= CREATE_NEW_CONSOLE;
@@ -790,6 +835,11 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine
}
}
+ for (const auto& [Key, Value] : Options.Environment)
+ {
+ setenv(Key.c_str(), Value.c_str(), 1);
+ }
+
if (execv(Executable.c_str(), ArgV.data()) < 0)
{
ThrowLastError("Failed to exec() a new process image");
diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp
index 58b76783a..b7d01003b 100644
--- a/src/zencore/sentryintegration.cpp
+++ b/src/zencore/sentryintegration.cpp
@@ -31,6 +31,8 @@ namespace {
struct SentryAssertImpl : zen::AssertImpl
{
+ ZEN_DEBUG_SECTION ~SentryAssertImpl() override = default;
+
virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename,
int LineNumber,
const char* FunctionName,
diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp
index 141450b84..8985a8a76 100644
--- a/src/zencore/system.cpp
+++ b/src/zencore/system.cpp
@@ -660,6 +660,24 @@ GetCpuName()
#endif
}
+std::string_view
+GetCompilerName()
+{
+#define ZEN_STRINGIFY_IMPL(x) #x
+#define ZEN_STRINGIFY(x) ZEN_STRINGIFY_IMPL(x)
+#if ZEN_COMPILER_CLANG
+ return "clang " ZEN_STRINGIFY(__clang_major__) "." ZEN_STRINGIFY(__clang_minor__) "." ZEN_STRINGIFY(__clang_patchlevel__);
+#elif ZEN_COMPILER_MSC
+ return "MSVC " ZEN_STRINGIFY(_MSC_VER);
+#elif ZEN_COMPILER_GCC
+ return "GCC " ZEN_STRINGIFY(__GNUC__) "." ZEN_STRINGIFY(__GNUC_MINOR__) "." ZEN_STRINGIFY(__GNUC_PATCHLEVEL__);
+#else
+ return "unknown";
+#endif
+#undef ZEN_STRINGIFY
+#undef ZEN_STRINGIFY_IMPL
+}
+
void
Describe(const SystemMetrics& Metrics, CbWriter& Writer)
{
diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp
index d7eb3b17d..f5bc723b1 100644
--- a/src/zencore/testing.cpp
+++ b/src/zencore/testing.cpp
@@ -24,6 +24,8 @@
# if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC
# include <execinfo.h>
# include <unistd.h>
+# elif ZEN_PLATFORM_WINDOWS
+# include <crtdbg.h>
# endif
namespace zen::testing {
@@ -296,6 +298,17 @@ RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink
}
# endif
+# if ZEN_PLATFORM_WINDOWS
+ // Suppress Windows error dialogs (crash/abort/assert) so tests terminate
+ // immediately instead of blocking on a modal dialog in CI or headless runs.
+ SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX);
+ _set_abort_behavior(0, _WRITE_ABORT_MSG);
+ _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE);
+ _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR);
+ _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE);
+ _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR);
+# endif
+
zen::logging::InitializeLogging();
zen::MaximizeOpenFileCount();
InstallCrashSignalHandlers();
diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp
index 5f3ad2455..3ca586f87 100644
--- a/src/zenhttp/httpclient_test.cpp
+++ b/src/zenhttp/httpclient_test.cpp
@@ -154,6 +154,42 @@ public:
},
HttpVerb::kGet);
+ m_Router.AddMatcher("anypath", [](std::string_view Str) -> bool { return !Str.empty(); });
+
+ m_Router.RegisterRoute(
+ "echo/uri",
+ [](HttpRouterRequest& Req) {
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ std::string Body = std::string(HttpReq.RelativeUri());
+
+ auto Params = HttpReq.GetQueryParams();
+ for (const auto& [Key, Value] : Params.KvPairs)
+ {
+ Body += fmt::format("\n{}={}", Key, Value);
+ }
+
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body);
+ },
+ HttpVerb::kGet | HttpVerb::kPut);
+
+ m_Router.RegisterRoute(
+ "echo/uri/{anypath}",
+ [](HttpRouterRequest& Req) {
+ // Echo both the RelativeUri and the captured path segment
+ HttpServerRequest& HttpReq = Req.ServerRequest();
+ std::string_view Captured = Req.GetCapture(1);
+ std::string Body = fmt::format("uri={}\ncapture={}", HttpReq.RelativeUri(), Captured);
+
+ auto Params = HttpReq.GetQueryParams();
+ for (const auto& [Key, Value] : Params.KvPairs)
+ {
+ Body += fmt::format("\n{}={}", Key, Value);
+ }
+
+ HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body);
+ },
+ HttpVerb::kGet | HttpVerb::kPut);
+
m_Router.RegisterRoute(
"slow",
[](HttpRouterRequest& Req) {
@@ -1689,6 +1725,77 @@ TEST_CASE("httpclient.https")
# endif // ZEN_USE_OPENSSL
+TEST_CASE("httpclient.uri_decoding")
+{
+ TestServerFixture Fixture;
+ HttpClient Client = Fixture.MakeClient();
+
+ // URI without encoding — should pass through unchanged
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello/world.txt");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "uri=echo/uri/hello/world.txt\ncapture=hello/world.txt");
+ }
+
+ // Percent-encoded space — server should see decoded path
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt");
+ }
+
+ // Percent-encoded slash (%2F) — should be decoded to /
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/a%2Fb.txt");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "uri=echo/uri/a/b.txt\ncapture=a/b.txt");
+ }
+
+ // Multiple encodings in one path
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/file%20%26%20name.txt");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "uri=echo/uri/file & name.txt\ncapture=file & name.txt");
+ }
+
+ // No capture — echo/uri route returns just RelativeUri
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "echo/uri");
+ }
+
+ // Literal percent that is not an escape (%ZZ) — should be kept as-is
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/100%25done.txt");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "uri=echo/uri/100%done.txt\ncapture=100%done.txt");
+ }
+
+ // Query params — raw values are returned as-is from GetQueryParams
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri?key=value&name=test");
+ REQUIRE(Resp.IsSuccess());
+ CHECK(Resp.AsText() == "echo/uri\nkey=value\nname=test");
+ }
+
+ // Query params with percent-encoded values
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri?prefix=listing%2F&mode=s3");
+ REQUIRE(Resp.IsSuccess());
+ // GetQueryParams returns raw (still-encoded) values — callers must Decode() explicitly
+ CHECK(Resp.AsText() == "echo/uri\nprefix=listing%2F\nmode=s3");
+ }
+
+ // Query params with path capture and encoding
+ {
+ HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt?tag=a%26b");
+ REQUIRE(Resp.IsSuccess());
+ // Path is decoded, query values are raw
+ CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt\ntag=a%26b");
+ }
+}
+
TEST_SUITE_END();
void
diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp
index 672467f56..4d98e9650 100644
--- a/src/zenhttp/httpserver.cpp
+++ b/src/zenhttp/httpserver.cpp
@@ -700,15 +700,6 @@ HttpServerRequest::ReadPayloadPackage()
//////////////////////////////////////////////////////////////////////////
void
-HttpRequestRouter::AddPattern(const char* Id, const char* Regex)
-{
- ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end());
- ZEN_ASSERT(!m_IsFinalized);
-
- m_PatternMap.insert({Id, Regex});
-}
-
-void
HttpRequestRouter::AddMatcher(const char* Id, std::function<bool(std::string_view)>&& Matcher)
{
ZEN_ASSERT(m_MatcherNameMap.find(Id) == m_MatcherNameMap.end());
@@ -724,170 +715,77 @@ HttpRequestRouter::RegisterRoute(const char* UriPattern, HttpRequestRouter::Hand
{
ZEN_ASSERT(!m_IsFinalized);
- if (ExtendableStringBuilder<128> ExpandedRegex; ProcessRegexSubstitutions(UriPattern, ExpandedRegex))
- {
- // Regex route
- m_RegexHandlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), UriPattern);
- }
- else
- {
- // New-style regex-free route. More efficient and should be used for everything eventually
+ int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern));
- int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern));
+ int i = 0;
- int i = 0;
+ std::vector<int> MatcherIndices;
- std::vector<int> MatcherIndices;
-
- while (i < RegexLen)
+ while (i < RegexLen)
+ {
+ if (UriPattern[i] == '{')
{
- if (UriPattern[i] == '{')
+ bool IsComplete = false;
+ int PatternStart = i + 1;
+ while (++i < RegexLen)
{
- bool IsComplete = false;
- int PatternStart = i + 1;
- while (++i < RegexLen)
+ if (UriPattern[i] == '}')
{
- if (UriPattern[i] == '}')
+ if (i == PatternStart)
{
- if (i == PatternStart)
- {
- throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern));
- }
- std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart);
- if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end())
- {
- // It's a match
- MatcherIndices.push_back(it->second);
- IsComplete = true;
- ++i;
- break;
- }
- else
- {
- throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern));
- }
+ throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern));
}
- }
- if (!IsComplete)
- {
- throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern));
- }
- }
- else
- {
- if (UriPattern[i] == '/')
- {
- throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern));
- }
-
- int SegmentStart = i;
- while (++i < RegexLen && UriPattern[i] != '/')
- ;
-
- std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart));
- int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size());
- m_Literals.push_back(std::string(Segment));
- MatcherIndices.push_back(-1 - LiteralIndex);
- }
-
- if (i < RegexLen && UriPattern[i] == '/')
- {
- ++i; // skip slash
- }
- }
-
- m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern);
- }
-}
-
-std::string_view
-HttpRouterRequest::GetCapture(uint32_t Index) const
-{
- if (!m_CapturedSegments.empty())
- {
- ZEN_ASSERT(Index < m_CapturedSegments.size());
- return m_CapturedSegments[Index];
- }
-
- ZEN_ASSERT(Index < m_Match.size());
-
- const auto& Match = m_Match[Index];
-
- return std::string_view(&*Match.first, Match.second - Match.first);
-}
-
-bool
-HttpRequestRouter::ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& OutExpandedRegex)
-{
- size_t RegexLen = strlen(Regex);
-
- bool HasRegex = false;
-
- std::vector<std::string> UnknownPatterns;
-
- for (size_t i = 0; i < RegexLen;)
- {
- bool matched = false;
-
- if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\')))
- {
- // Might have a pattern reference - find closing brace
-
- for (size_t j = i + 1; j < RegexLen; ++j)
- {
- if (Regex[j] == '}')
- {
- std::string Pattern(&Regex[i + 1], j - i - 1);
-
- if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end())
+ std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart);
+ if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end())
{
- OutExpandedRegex.Append(it->second.c_str());
- HasRegex = true;
+ // It's a match
+ MatcherIndices.push_back(it->second);
+ IsComplete = true;
+ ++i;
+ break;
}
else
{
- UnknownPatterns.push_back(Pattern);
+ throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern));
}
-
- // skip ahead
- i = j + 1;
-
- matched = true;
-
- break;
}
}
+ if (!IsComplete)
+ {
+ throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern));
+ }
}
-
- if (!matched)
- {
- OutExpandedRegex.Append(Regex[i++]);
- }
- }
-
- if (HasRegex)
- {
- if (UnknownPatterns.size() > 0)
+ else
{
- std::string UnknownList;
- for (const auto& Pattern : UnknownPatterns)
+ if (UriPattern[i] == '/')
{
- if (!UnknownList.empty())
- {
- UnknownList += ", ";
- }
- UnknownList += "'";
- UnknownList += Pattern;
- UnknownList += "'";
+ throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern));
}
- throw std::runtime_error(fmt::format("unknown pattern(s) {} in regex route '{}'", UnknownList, Regex));
+ int SegmentStart = i;
+ while (++i < RegexLen && UriPattern[i] != '/')
+ ;
+
+ std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart));
+ int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size());
+ m_Literals.push_back(std::string(Segment));
+ MatcherIndices.push_back(-1 - LiteralIndex);
}
- return true;
+ if (i < RegexLen && UriPattern[i] == '/')
+ {
+ ++i; // skip slash
+ }
}
- return false;
+ m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern);
+}
+
+std::string_view
+HttpRouterRequest::GetCapture(uint32_t Index) const
+{
+ ZEN_ASSERT(Index < m_CapturedSegments.size());
+ return m_CapturedSegments[Index];
}
bool
@@ -903,8 +801,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
std::string_view Uri = Request.RelativeUri();
HttpRouterRequest RouterRequest(Request);
- // First try new-style matcher routes
-
for (const MatcherEndpoint& Handler : m_MatcherEndpoints)
{
if ((Handler.Verbs & Verb) == Verb)
@@ -1002,28 +898,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request)
}
}
- // Old-style regex routes
-
- for (const auto& Handler : m_RegexHandlers)
- {
- if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx))
- {
-#if ZEN_WITH_OTEL
- if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan())
- {
- ExtendableStringBuilder<128> RoutePath;
- RoutePath.Append(Request.Service().BaseUri());
- RoutePath.Append(Handler.Pattern);
- ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView());
- }
-#endif
-
- Handler.Handler(RouterRequest);
-
- return true; // Route matched
- }
- }
-
return false; // No route matched
}
@@ -1422,72 +1296,6 @@ TEST_CASE("http.common")
virtual uint32_t ParseRequestId() const override { return 0; }
};
- SUBCASE("router-regex")
- {
- bool HandledA = false;
- bool HandledAA = false;
- std::vector<std::string> Captures;
- auto Reset = [&] {
- Captures.clear();
- HandledA = HandledAA = false;
- };
-
- TestHttpService Service;
-
- HttpRequestRouter r;
- r.AddPattern("a", "([[:alpha:]]+)");
- r.RegisterRoute(
- "{a}",
- [&](auto& Req) {
- HandledA = true;
- Captures = {std::string(Req.GetCapture(0))};
- },
- HttpVerb::kGet);
-
- r.RegisterRoute(
- "{a}/{a}",
- [&](auto& Req) {
- HandledAA = true;
- Captures = {std::string(Req.GetCapture(1)), std::string(Req.GetCapture(2))};
- },
- HttpVerb::kGet);
-
- {
- Reset();
- TestHttpServerRequest req(Service, "abc"sv);
- r.HandleRequest(req);
- CHECK(HandledA);
- CHECK(!HandledAA);
- REQUIRE_EQ(Captures.size(), 1);
- CHECK_EQ(Captures[0], "abc"sv);
- }
-
- {
- Reset();
- TestHttpServerRequest req{Service, "abc/def"sv};
- r.HandleRequest(req);
- CHECK(!HandledA);
- CHECK(HandledAA);
- REQUIRE_EQ(Captures.size(), 2);
- CHECK_EQ(Captures[0], "abc"sv);
- CHECK_EQ(Captures[1], "def"sv);
- }
-
- {
- Reset();
- TestHttpServerRequest req{Service, "123"sv};
- r.HandleRequest(req);
- CHECK(!HandledA);
- }
-
- {
- Reset();
- TestHttpServerRequest req{Service, "a123"sv};
- r.HandleRequest(req);
- CHECK(!HandledA);
- }
- }
-
SUBCASE("router-matcher")
{
bool HandledA = false;
diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h
index 77feb6568..2a8b2ca94 100644
--- a/src/zenhttp/include/zenhttp/httpserver.h
+++ b/src/zenhttp/include/zenhttp/httpserver.h
@@ -20,7 +20,6 @@
#include <gsl/gsl-lite.hpp>
#include <list>
#include <map>
-#include <regex>
#include <span>
#include <unordered_map>
@@ -357,9 +356,8 @@ class HttpRouterRequest
public:
/** Get captured segment from matched URL
*
- * @param Index Index of captured segment to retrieve. Note that due to
- * backwards compatibility with regex-based routes, this index is 1-based
- * and index=0 is the full matched URL
+ * @param Index Index of captured segment to retrieve. Index 0 is the full
+ * matched URL, subsequent indices are the matched segments in order.
* @return Returns string view of captured segment
*/
std::string_view GetCapture(uint32_t Index) const;
@@ -372,11 +370,8 @@ private:
HttpRouterRequest(const HttpRouterRequest&) = delete;
HttpRouterRequest& operator=(const HttpRouterRequest&) = delete;
- using MatchResults_t = std::match_results<std::string_view::const_iterator>;
-
HttpServerRequest& m_HttpRequest;
- MatchResults_t m_Match;
- std::vector<std::string_view> m_CapturedSegments; // for matcher-based routes
+ std::vector<std::string_view> m_CapturedSegments;
friend class HttpRequestRouter;
};
@@ -384,9 +379,7 @@ private:
/** HTTP request router helper
*
* This helper class allows a service implementer to register one or more
- * endpoints using pattern matching. We currently support a legacy regex-based
- * matching system, but also a new matcher-function based system which is more
- * efficient and should be used whenever possible.
+ * endpoints using pattern matching with matcher functions.
*
* This is intended to be initialized once only, there is no thread
* safety so you can absolutely not add or remove endpoints once the handler
@@ -405,13 +398,6 @@ public:
typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t;
/**
- * @brief Add pattern which can be referenced by name, commonly used for URL components
- * @param Id String used to identify patterns for replacement
- * @param Regex String which will replace the Id string in any registered URL paths
- */
- void AddPattern(const char* Id, const char* Regex);
-
- /**
* @brief Add matcher function which can be referenced by name, used for URL components
* @param Id String used to identify matchers in endpoint specifications
* @param Matcher Function which will be called to match the component
@@ -421,8 +407,8 @@ public:
/**
* @brief Register an endpoint handler for the given route
* @param Pattern Pattern used to match the handler to a request. This should
- * only contain literal URI segments and pattern aliases registered
- via AddPattern() or AddMatcher()
+ * only contain literal URI segments and matcher aliases registered
+ via AddMatcher()
* @param HandlerFunc Handler function to call for any matching request
* @param SupportedVerbs Supported HTTP verbs for this handler
*/
@@ -437,36 +423,6 @@ public:
bool HandleRequest(zen::HttpServerRequest& Request);
private:
- bool ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& ExpandedRegex);
-
- struct RegexEndpoint
- {
- RegexEndpoint(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern)
- : RegEx(Regex, std::regex::icase | std::regex::ECMAScript)
- , Verbs(SupportedVerbs)
- , Handler(std::move(Handler))
- , Pattern(Pattern)
- {
- }
-
- ~RegexEndpoint() = default;
-
- std::regex RegEx;
- HttpVerb Verbs;
- HandlerFunc_t Handler;
- const char* Pattern;
-
- private:
- RegexEndpoint& operator=(const RegexEndpoint&) = delete;
- RegexEndpoint(const RegexEndpoint&) = delete;
- };
-
- std::list<RegexEndpoint> m_RegexHandlers;
- std::unordered_map<std::string, std::string> m_PatternMap;
-
- // New-style matcher endpoints. Should be preferred over regex endpoints where possible
- // as it is considerably more efficient
-
struct MatcherEndpoint
{
MatcherEndpoint(std::vector<int>&& ComponentIndices, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern)
diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp
index 643f33618..9f4875eaf 100644
--- a/src/zenhttp/servers/httpasio.cpp
+++ b/src/zenhttp/servers/httpasio.cpp
@@ -601,6 +601,7 @@ public:
bool m_IsLocalMachineRequest;
bool m_AllowZeroCopyFileSend = true;
std::string m_RemoteAddress;
+ std::string m_DecodedUri; // Percent-decoded URI; m_Uri/m_UriWithExtension point into this
std::unique_ptr<HttpResponse> m_Response;
};
@@ -623,6 +624,7 @@ public:
~HttpResponse() = default;
void SetAllowZeroCopyFileSend(bool Allow) { m_AllowZeroCopyFileSend = Allow; }
+ void SetKeepAlive(bool KeepAlive) { m_IsKeepAlive = KeepAlive; }
/**
* Initialize the response for sending a payload made up of multiple blobs
@@ -780,8 +782,8 @@ public:
return m_Headers;
}
- template<typename SocketType>
- void SendResponse(SocketType& Socket, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token)
+ template<typename SocketType, typename Executor>
+ void SendResponse(SocketType& Socket, Executor& Strand, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token)
{
ZEN_ASSERT(m_State == State::kInitialized);
@@ -791,11 +793,11 @@ public:
m_SendCb = std::move(Token);
m_State = State::kSending;
- SendNextChunk(Socket);
+ SendNextChunk(Socket, Strand);
}
- template<typename SocketType>
- void SendNextChunk(SocketType& Socket)
+ template<typename SocketType, typename Executor>
+ void SendNextChunk(SocketType& Socket, Executor& Strand)
{
ZEN_ASSERT(m_State == State::kSending);
@@ -812,12 +814,12 @@ public:
auto CompletionToken = [Self = this, Token = std::move(m_SendCb), TotalBytes = m_TotalBytesSent] { Token({}, TotalBytes); };
- asio::defer(Socket.get_executor(), std::move(CompletionToken));
+ asio::defer(Strand, std::move(CompletionToken));
return;
}
- auto OnCompletion = [this, &Socket](const asio::error_code& Ec, std::size_t ByteCount) {
+ auto OnCompletion = asio::bind_executor(Strand, [this, &Socket, &Strand](const asio::error_code& Ec, std::size_t ByteCount) {
ZEN_ASSERT(m_State == State::kSending);
m_TotalBytesSent += ByteCount;
@@ -828,9 +830,9 @@ public:
}
else
{
- SendNextChunk(Socket);
+ SendNextChunk(Socket, Strand);
}
- };
+ });
const IoVec& Io = m_IoVecs[m_IoVecCursor++];
@@ -982,16 +984,14 @@ private:
void CloseConnection();
void SendInlineResponse(uint32_t RequestNumber, std::string_view StatusLine, std::string_view Headers = {}, std::string_view Body = {});
- HttpAsioServerImpl& m_Server;
- asio::streambuf m_RequestBuffer;
- std::atomic<uint32_t> m_RequestCounter{0};
- uint32_t m_ConnectionId = 0;
- Ref<IHttpPackageHandler> m_PackageHandler;
-
- RwLock m_ActiveResponsesLock;
+ HttpAsioServerImpl& m_Server;
+ std::unique_ptr<SocketType> m_Socket;
+ asio::strand<asio::any_io_executor> m_Strand;
+ asio::streambuf m_RequestBuffer;
+ uint32_t m_RequestCounter = 0;
+ uint32_t m_ConnectionId = 0;
+ Ref<IHttpPackageHandler> m_PackageHandler;
std::deque<std::unique_ptr<HttpResponse>> m_ActiveResponses;
-
- std::unique_ptr<SocketType> m_Socket;
};
std::atomic<uint32_t> g_ConnectionIdCounter{0};
@@ -999,8 +999,9 @@ std::atomic<uint32_t> g_ConnectionIdCounter{0};
template<typename SocketType>
HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Server, std::unique_ptr<SocketType>&& Socket)
: m_Server(Server)
-, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1))
, m_Socket(std::move(Socket))
+, m_Strand(asio::make_strand(m_Socket->get_executor()))
+, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1))
{
ZEN_TRACE_VERBOSE("new connection #{}", m_ConnectionId);
}
@@ -1008,8 +1009,6 @@ HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Ser
template<typename SocketType>
HttpServerConnectionT<SocketType>::~HttpServerConnectionT()
{
- RwLock::ExclusiveLockScope _(m_ActiveResponsesLock);
-
ZEN_TRACE_VERBOSE("destroying connection #{}", m_ConnectionId);
}
@@ -1017,7 +1016,7 @@ template<typename SocketType>
void
HttpServerConnectionT<SocketType>::HandleNewRequest()
{
- EnqueueRead();
+ asio::dispatch(m_Strand, [Conn = AsSharedPtr()] { Conn->EnqueueRead(); });
}
template<typename SocketType>
@@ -1058,7 +1057,9 @@ HttpServerConnectionT<SocketType>::EnqueueRead()
asio::async_read(*m_Socket.get(),
m_RequestBuffer,
asio::transfer_at_least(1),
- [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnDataReceived(Ec, ByteCount); });
+ asio::bind_executor(m_Strand, [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) {
+ Conn->OnDataReceived(Ec, ByteCount);
+ }));
}
template<typename SocketType>
@@ -1091,7 +1092,7 @@ HttpServerConnectionT<SocketType>::OnDataReceived(const asio::error_code& Ec, [[
ZEN_TRACE_VERBOSE("on data received, connection: {}, request: {}, thread: {}, bytes: {}",
m_ConnectionId,
- m_RequestCounter.load(std::memory_order_relaxed),
+ m_RequestCounter,
zen::GetCurrentThreadId(),
NiceBytes(ByteCount));
@@ -1153,25 +1154,23 @@ HttpServerConnectionT<SocketType>::OnResponseDataSent(const asio::error_code&
if (ResponseToPop)
{
- m_ActiveResponsesLock.WithExclusiveLock([&] {
- // Once a response is sent we can release any referenced resources
- //
- // completion callbacks may be issued out-of-order so we need to
- // remove the relevant entry from our active response list, it may
- // not be the first
-
- if (auto It = find_if(begin(m_ActiveResponses),
- end(m_ActiveResponses),
- [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; });
- It != end(m_ActiveResponses))
- {
- m_ActiveResponses.erase(It);
- }
- else
- {
- ZEN_WARN("response not found");
- }
- });
+ // Once a response is sent we can release any referenced resources
+ //
+ // completion callbacks may be issued out-of-order so we need to
+ // remove the relevant entry from our active response list, it may
+ // not be the first
+
+ if (auto It = find_if(begin(m_ActiveResponses),
+ end(m_ActiveResponses),
+ [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; });
+ It != end(m_ActiveResponses))
+ {
+ m_ActiveResponses.erase(It);
+ }
+ else
+ {
+ ZEN_WARN("response not found");
+ }
}
if (!m_RequestData.IsKeepAlive())
@@ -1234,9 +1233,11 @@ HttpServerConnectionT<SocketType>::SendInlineResponse(uint32_t RequestNumber
asio::async_write(
*m_Socket,
Buffer,
- [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) {
- Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr);
- });
+ asio::bind_executor(
+ m_Strand,
+ [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) {
+ Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr);
+ }));
}
template<typename SocketType>
@@ -1272,21 +1273,23 @@ HttpServerConnectionT<SocketType>::HandleRequest()
asio::async_write(
*m_Socket,
asio::buffer(ResponseStr->data(), ResponseStr->size()),
- [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) {
- if (Ec)
- {
- ZEN_WARN("WebSocket 101 send failed: {}", Ec.message());
- return;
- }
-
- Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened();
- using WsConnType = WsAsioConnectionT<SocketType>;
- Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer));
- Ref<WebSocketConnection> WsConnRef(WsConn.Get());
-
- WsHandler->OnWebSocketOpen(std::move(WsConnRef));
- WsConn->Start();
- });
+ asio::bind_executor(
+ m_Strand,
+ [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) {
+ if (Ec)
+ {
+ ZEN_WARN("WebSocket 101 send failed: {}", Ec.message());
+ return;
+ }
+
+ Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened();
+ using WsConnType = WsAsioConnectionT<SocketType>;
+ Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer));
+ Ref<WebSocketConnection> WsConnRef(WsConn.Get());
+
+ WsHandler->OnWebSocketOpen(std::move(WsConnRef));
+ WsConn->Start();
+ }));
m_RequestState = RequestState::kDone;
return;
@@ -1312,7 +1315,7 @@ HttpServerConnectionT<SocketType>::HandleRequest()
m_RequestState = RequestState::kWriting;
}
- const uint32_t RequestNumber = m_RequestCounter.fetch_add(1);
+ const uint32_t RequestNumber = m_RequestCounter++;
if (HttpService* Service = m_Server.RouteRequest(m_RequestData.Url()))
{
@@ -1444,31 +1447,34 @@ HttpServerConnectionT<SocketType>::HandleRequest()
{
ZEN_TRACE_CPU("asio::async_write");
- std::string_view Headers = Response->GetHeaders();
+ HttpResponse* ResponseRaw = Response.get();
+ m_ActiveResponses.push_back(std::move(Response));
+
+ std::string_view Headers = ResponseRaw->GetHeaders();
std::vector<asio::const_buffer> AsioBuffers;
AsioBuffers.push_back(asio::const_buffer(Headers.data(), Headers.size()));
- asio::async_write(*m_Socket.get(),
- AsioBuffers,
- asio::transfer_all(),
- [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) {
- Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr);
- });
+ asio::async_write(
+ *m_Socket.get(),
+ AsioBuffers,
+ asio::transfer_all(),
+ asio::bind_executor(
+ m_Strand,
+ [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) {
+ Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw);
+ }));
}
else
{
ZEN_TRACE_CPU("asio::async_write");
HttpResponse* ResponseRaw = Response.get();
-
- m_ActiveResponsesLock.WithExclusiveLock([&] {
- // Keep referenced resources alive
- m_ActiveResponses.push_back(std::move(Response));
- });
+ m_ActiveResponses.push_back(std::move(Response));
ResponseRaw->SendResponse(
*m_Socket,
+ m_Strand,
[Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) {
Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw);
});
@@ -1982,11 +1988,24 @@ HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request,
{
const int PrefixLength = Service.UriPrefixLength();
- std::string_view Uri = Request.Url();
- Uri.remove_prefix(std::min(PrefixLength, static_cast<int>(Uri.size())));
- m_Uri = Uri;
- m_UriWithExtension = Uri;
- m_QueryString = Request.QueryString();
+ std::string_view RawUri = Request.Url();
+ RawUri.remove_prefix(std::min(PrefixLength, static_cast<int>(RawUri.size())));
+
+ // Percent-decode the URI path so handlers see the same decoded paths regardless
+ // of whether the ASIO or http.sys backend is used (http.sys pre-decodes via CookedUrl).
+ // Skip the allocation when there is nothing to decode (common case).
+ if (RawUri.find('%') != std::string_view::npos)
+ {
+ m_DecodedUri = Decode(RawUri);
+ m_Uri = m_DecodedUri;
+ m_UriWithExtension = m_DecodedUri;
+ }
+ else
+ {
+ m_Uri = RawUri;
+ m_UriWithExtension = RawUri;
+ }
+ m_QueryString = Request.QueryString();
m_Verb = Request.RequestVerb();
m_ContentLength = Request.Body().Size();
@@ -2083,6 +2102,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode)
m_Response.reset(new HttpResponse(HttpContentType::kBinary, m_RequestNumber));
m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend);
+ m_Response->SetKeepAlive(m_Request.IsKeepAlive());
std::array<IoBuffer, 0> Empty;
m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty);
@@ -2097,6 +2117,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT
m_Response.reset(new HttpResponse(ContentType, m_RequestNumber));
m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend);
+ m_Response->SetKeepAlive(m_Request.IsKeepAlive());
m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs);
}
@@ -2108,6 +2129,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT
ZEN_ASSERT(!m_Response);
m_Response.reset(new HttpResponse(ContentType, m_RequestNumber));
m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend);
+ m_Response->SetKeepAlive(m_Request.IsKeepAlive());
IoBuffer MessageBuffer(IoBuffer::Wrap, ResponseString.data(), ResponseString.size());
std::array<IoBuffer, 1> SingleBufferList({MessageBuffer});
diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp
index a02ca7be3..0d8550c5b 100644
--- a/src/zenserver/compute/computeserver.cpp
+++ b/src/zenserver/compute/computeserver.cpp
@@ -865,7 +865,7 @@ ZenComputeServer::Run()
ExtendableStringBuilder<256> BuildOptions;
GetBuildOptions(BuildOptions, '\n');
- ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+ ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions);
}
ZEN_INFO(ZEN_APP_NAME " now running as COMPUTE (pid: {})", GetCurrentProcessId());
diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp
index 696991403..b0ae0a8b1 100644
--- a/src/zenserver/hub/zenhubserver.cpp
+++ b/src/zenserver/hub/zenhubserver.cpp
@@ -337,7 +337,7 @@ ZenHubServer::Run()
ExtendableStringBuilder<256> BuildOptions;
GetBuildOptions(BuildOptions, '\n');
- ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+ ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions);
}
ZEN_INFO(ZEN_APP_NAME " now running as HUB (pid: {})", GetCurrentProcessId());
diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp
index acfdad45f..c768e940a 100644
--- a/src/zenserver/proxy/zenproxyserver.cpp
+++ b/src/zenserver/proxy/zenproxyserver.cpp
@@ -359,7 +359,7 @@ ZenProxyServer::Run()
ExtendableStringBuilder<256> BuildOptions;
GetBuildOptions(BuildOptions, '\n');
- ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+ ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions);
}
ZEN_INFO(ZEN_APP_NAME " now running as PROXY (pid: {})", GetCurrentProcessId());
diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp
index 052c3d630..e347e2dfe 100644
--- a/src/zenserver/storage/objectstore/objectstore.cpp
+++ b/src/zenserver/storage/objectstore/objectstore.cpp
@@ -271,7 +271,7 @@ HttpObjectStoreService::Inititalize()
CreateDirectories(BucketsPath);
}
- static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
+ static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]() ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"};
m_Router.AddMatcher("path",
@@ -292,10 +292,9 @@ HttpObjectStoreService::Inititalize()
m_Router.RegisterRoute(
"bucket/{path}",
[this](zen::HttpRouterRequest& Request) {
- const std::string_view EncodedPath = Request.GetCapture(1);
- const std::string Path = Request.ServerRequest().Decode(EncodedPath);
- const auto Sep = Path.find_last_of('.');
- const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0;
+ const std::string_view Path = Request.GetCapture(1);
+ const auto Sep = Path.find_last_of('.');
+ const bool IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0;
if (IsObject)
{
@@ -378,7 +377,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s
const auto QueryParms = Request.ServerRequest().GetQueryParams();
if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false)
{
- BucketPrefix = PrefixParam;
+ BucketPrefix = HttpServerRequest::Decode(PrefixParam);
}
}
BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/'));
diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp
index 77588bd6c..bba5e0a61 100644
--- a/src/zenserver/storage/zenstorageserver.cpp
+++ b/src/zenserver/storage/zenstorageserver.cpp
@@ -720,7 +720,7 @@ ZenStorageServer::Run()
ExtendableStringBuilder<256> BuildOptions;
GetBuildOptions(BuildOptions, '\n');
- ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions);
+ ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions);
}
ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId());
diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp
index 519176ffe..6760e0372 100644
--- a/src/zenserver/zenserver.cpp
+++ b/src/zenserver/zenserver.cpp
@@ -201,6 +201,9 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState::
std::chrono::system_clock::now().time_since_epoch()).count(),
.BuildOptions = {
{"ZEN_ADDRESS_SANITIZER", ZEN_ADDRESS_SANITIZER != 0},
+ {"ZEN_THREAD_SANITIZER", ZEN_THREAD_SANITIZER != 0},
+ {"ZEN_MEMORY_SANITIZER", ZEN_MEMORY_SANITIZER != 0},
+ {"ZEN_LEAK_SANITIZER", ZEN_LEAK_SANITIZER != 0},
{"ZEN_USE_SENTRY", ZEN_USE_SENTRY != 0},
{"ZEN_WITH_TESTS", ZEN_WITH_TESTS != 0},
{"ZEN_USE_MIMALLOC", ZEN_USE_MIMALLOC != 0},
@@ -251,6 +254,12 @@ ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) co
OutOptions << "ZEN_ADDRESS_SANITIZER=" << (ZEN_ADDRESS_SANITIZER ? "1" : "0");
OutOptions << Separator;
+ OutOptions << "ZEN_THREAD_SANITIZER=" << (ZEN_THREAD_SANITIZER ? "1" : "0");
+ OutOptions << Separator;
+ OutOptions << "ZEN_MEMORY_SANITIZER=" << (ZEN_MEMORY_SANITIZER ? "1" : "0");
+ OutOptions << Separator;
+ OutOptions << "ZEN_LEAK_SANITIZER=" << (ZEN_LEAK_SANITIZER ? "1" : "0");
+ OutOptions << Separator;
OutOptions << "ZEN_USE_SENTRY=" << (ZEN_USE_SENTRY ? "1" : "0");
OutOptions << Separator;
OutOptions << "ZEN_WITH_TESTS=" << (ZEN_WITH_TESTS ? "1" : "0");
diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp
index b3450b805..f3edf804d 100644
--- a/src/zenstore/gc.cpp
+++ b/src/zenstore/gc.cpp
@@ -1776,11 +1776,13 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config)
m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64()));
m_LastGcExpireTime =
GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count())));
- if (m_LastGcTime + m_Config.Interval < GcClock::Now())
+ if (m_LastGcTime > GcClock::Now() || m_LastGcTime + m_Config.Interval < GcClock::Now())
{
- // TODO: Trigger GC?
+ // Reset if the stored timestamp is in the future (e.g. clock resolution mismatch
+ // between the build that wrote gc_state and this build) or too far in the past.
m_LastGcTime = GcClock::Now();
m_LastLightweightGcTime = m_LastGcTime;
+ m_LastGcExpireTime = GcClock::TimePoint::min();
}
m_AttachmentPassIndex = SchedulerState["AttachmentPassIndex"sv].AsUInt8();
}
@@ -2084,6 +2086,10 @@ GcScheduler::GetState() const
{
Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero();
}
+ else if (Result.RemainingTimeUntilFullGc > Result.Config.Interval)
+ {
+ Result.RemainingTimeUntilFullGc = Result.Config.Interval;
+ }
Result.RemainingTimeUntilLightweightGc =
Result.Config.LightweightInterval.count() == 0
@@ -2094,6 +2100,10 @@ GcScheduler::GetState() const
{
Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
}
+ else if (Result.RemainingTimeUntilLightweightGc > Result.Config.LightweightInterval)
+ {
+ Result.RemainingTimeUntilLightweightGc = Result.Config.LightweightInterval;
+ }
}
return Result;
@@ -2418,6 +2428,10 @@ GcScheduler::SchedulerThread()
{
RemainingTimeUntilGc = std::chrono::seconds::zero();
}
+ else if (RemainingTimeUntilGc > GcInterval)
+ {
+ RemainingTimeUntilGc = GcInterval;
+ }
std::chrono::seconds RemainingTimeUntilLightweightGc =
LightweightGcInterval.count() == 0 ? std::chrono::seconds::max()
@@ -2428,6 +2442,10 @@ GcScheduler::SchedulerThread()
{
RemainingTimeUntilLightweightGc = std::chrono::seconds::zero();
}
+ else if (RemainingTimeUntilLightweightGc > LightweightGcInterval)
+ {
+ RemainingTimeUntilLightweightGc = LightweightGcInterval;
+ }
// Don't schedule a lightweight GC if a full GC is
// due quite soon anyway
diff --git a/tsan.supp b/tsan.supp
index da8963854..93398da67 100644
--- a/tsan.supp
+++ b/tsan.supp
@@ -20,3 +20,15 @@
# TSAN reports as a race. This is benign: the slot is always NULL and writing NULL
# to it has no observable effect.
race:eastl::hashtable*DoFreeNodes*
+
+# UE::Trace's GetUid() uses a racy static uint32 cache (Uid = Uid ? Uid : Initialize())
+# as a fast path to avoid re-entering Initialize(). The actual initialization is done via
+# a thread-safe static (Uid_ThreadSafeInit) inside Initialize(), so the worst case is
+# redundant calls to Initialize() which always returns the same value.
+race:*Fields::GetUid*
+
+# TRACE_CPU_SCOPE generates a function-local `static int32 scope_id` that is lazily
+# initialized without synchronization (if (0 == scope_id) scope_id = ScopeNew(...)).
+# Same benign pattern as GetUid: the worst case is redundant calls to ScopeNew() which
+# always returns the same value for a given scope name.
+race:*$trace_scope_id*
diff --git a/xmake.lua b/xmake.lua
index acab78600..49dd3058f 100644
--- a/xmake.lua
+++ b/xmake.lua
@@ -48,12 +48,27 @@ set_policy("build.sanitizer.address", use_asan)
-- ThreadSanitizer, MemorySanitizer, LeakSanitizer, and UndefinedBehaviorSanitizer
-- are supported on Linux and MacOS only.
+--
+-- You can enable these by editing the xmake.lua directly, or by passing the
+-- appropriate flags on the command line:
+--
+-- `xmake --policies=build.sanitizer.thread:y` for ThreadSanitizer,
+-- `xmake --policies=build.sanitizer.memory:y` for MemorySanitizer, etc.
---set_policy("build.sanitizer.thread", true)
---set_policy("build.sanitizer.memory", true)
+-- When using TSAN you will want to also use the suppression tile to silence
+-- known benign races. You do this by ensuring the the TSAN_OPTIONS environment
+-- vriable is set to something like `TSAN_OPTIONS="suppressions=$(projectdir)/tsan.supp"`
+--
+-- `prompt> TSAN_OPTIONS="detect_deadlocks=0 suppressions=$(projectdir)/tsan.supp" xmake run zenserver`
+
+--set_policy("build.sanitizer.thread", true)
--set_policy("build.sanitizer.leak", true)
--set_policy("build.sanitizer.undefined", true)
+-- In practice, this does not work because of the difficulty of compiling
+-- dependencies with MemorySanitizer.
+--set_policy("build.sanitizer.memory", true)
+
--------------------------------------------------------------------------
-- Dependencies
@@ -470,401 +485,6 @@ task("test")
}
}
on_run(function()
- import("core.base.option")
- import("core.project.config")
- import("core.project.project")
-
- config.load()
-
- -- Override table: target name -> short name (for targets that don't follow convention)
- local short_name_overrides = {
- ["zenserver-test"] = "integration",
- }
-
- -- Build test list from targets in the "tests" group
- local available_tests = {}
- for name, target in pairs(project.targets()) do
- if target:get("group") == "tests" and name:endswith("-test") then
- local short = short_name_overrides[name]
- if not short then
- -- Derive short name: "zencore-test" -> "core"
- short = name
- if short:startswith("zen") then short = short:sub(4) end
- if short:endswith("-test") then short = short:sub(1, -6) end
- end
- table.insert(available_tests, {short, name})
- end
- end
-
- -- Add non-test-group entries that have a test subcommand
- table.insert(available_tests, {"server", "zenserver"})
-
- table.sort(available_tests, function(a, b) return a[1] < b[1] end)
-
- -- Handle --list: print discovered test names and exit
- if option.get("list") then
- printf("Available tests:\n")
- for _, entry in ipairs(available_tests) do
- printf(" %-16s -> %s\n", entry[1], entry[2])
- end
- return
- end
-
- local testname = option.get("run")
-
- -- Parse comma-separated test names into a set
- local requested = {}
- for token in testname:gmatch("[^,]+") do
- requested[token:match("^%s*(.-)%s*$")] = true
- end
-
- -- Filter to requested test(s)
- local tests = {}
- local matched = {}
-
- for _, entry in ipairs(available_tests) do
- local name, target = entry[1], entry[2]
- if requested["all"] or requested[name] then
- table.insert(tests, {name = name, target = target})
- matched[name] = true
- end
- end
-
- -- Check for unknown test names
- if not requested["all"] then
- for name, _ in pairs(requested) do
- if not matched[name] then
- raise("no tests match specification: '%s'", name)
- end
- end
- end
-
- if #tests == 0 then
- raise("no tests match specification: '%s'", testname)
- end
-
- local plat, arch
- if is_host("windows") then
- plat = "windows"
- arch = "x64"
- elseif is_host("macosx") then
- plat = "macosx"
- arch = is_arch("arm64") and "arm64" or "x86_64"
- else
- plat = "linux"
- arch = "x86_64"
- end
-
- -- Only reconfigure if current config doesn't already match
- if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then
- local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or ""
- local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or ""
- os.exec("xmake config -y -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag)
- end
-
- -- Build targets we're going to run
- if requested["all"] then
- os.exec("xmake build -y")
- else
- for _, entry in ipairs(tests) do
- os.exec("xmake build -y %s", entry.target)
- end
- end
-
- local use_junit_reporting = option.get("junit")
- local use_noskip = option.get("noskip")
- local use_verbose = option.get("verbose")
- local repeat_count = tonumber(option.get("repeat")) or 1
- local extra_args = option.get("arguments") or {}
- local junit_report_files = {}
-
- local junit_report_dir
- if use_junit_reporting then
- junit_report_dir = path.join(os.projectdir(), config.get("buildir"), "reports")
- os.mkdir(junit_report_dir)
- end
-
- -- Results collection for summary table
- local results = {}
- local any_failed = false
-
- -- Format a number with thousands separators (e.g. 31103 -> "31,103")
- local function format_number(n)
- local s = tostring(n)
- local pos = #s % 3
- if pos == 0 then pos = 3 end
- local result = s:sub(1, pos)
- for i = pos + 1, #s, 3 do
- result = result .. "," .. s:sub(i, i + 2)
- end
- return result
- end
-
- -- Center a string within a given width
- local function center_str(s, width)
- local pad = width - #s
- local lpad = math.floor(pad / 2)
- local rpad = pad - lpad
- return string.rep(" ", lpad) .. s .. string.rep(" ", rpad)
- end
-
- -- Left-align a string within a given width (with 1-space left margin)
- local function left_align_str(s, width)
- return " " .. s .. string.rep(" ", width - #s - 1)
- end
-
- -- Right-align a string within a given width (with 1-space right margin)
- local function right_align_str(s, width)
- return string.rep(" ", width - #s - 1) .. s .. " "
- end
-
- -- Format elapsed seconds as a human-readable string
- local function format_time(seconds)
- if seconds >= 60 then
- local mins = math.floor(seconds / 60)
- local secs = seconds - mins * 60
- return string.format("%dm %04.1fs", mins, secs)
- else
- return string.format("%.1fs", seconds)
- end
- end
-
- -- Parse test summary file written by TestListener
- local function parse_summary_file(filepath)
- if not os.isfile(filepath) then return nil end
- local content = io.readfile(filepath)
- if not content then return nil end
- local ct = content:match("cases_total=(%d+)")
- local cp = content:match("cases_passed=(%d+)")
- local at = content:match("assertions_total=(%d+)")
- local ap = content:match("assertions_passed=(%d+)")
- if ct then
- local failures = {}
- for name, file, line in content:gmatch("failed=([^|\n]+)|([^|\n]+)|(%d+)") do
- table.insert(failures, {name = name, file = file, line = tonumber(line)})
- end
- local es = content:match("elapsed_seconds=([%d%.]+)")
- return {
- cases_total = tonumber(ct),
- cases_passed = tonumber(cp) or 0,
- asserts_total = tonumber(at) or 0,
- asserts_passed = tonumber(ap) or 0,
- elapsed_seconds = tonumber(es) or 0,
- failures = failures
- }
- end
- return nil
- end
-
- -- Temp directory for summary files
- local summary_dir = path.join(os.tmpdir(), "zen-test-summary")
- os.mkdir(summary_dir)
-
- -- Run each test suite and collect results
- for iteration = 1, repeat_count do
- if repeat_count > 1 then
- printf("\n*** Iteration %d/%d ***\n", iteration, repeat_count)
- end
-
- for _, entry in ipairs(tests) do
- local name, target = entry.name, entry.target
- printf("=== %s ===\n", target)
-
- local suite_name = target
- if name == "server" then
- suite_name = "zenserver (test)"
- end
-
- local cmd = string.format("xmake run %s", target)
- if name == "server" then
- cmd = string.format("xmake run %s test", target)
- end
- cmd = string.format("%s --duration=true", cmd)
-
- if use_junit_reporting then
- local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target))
- junit_report_files[target] = junit_report_file
- cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file)
- end
- if use_noskip then
- cmd = string.format("%s --no-skip", cmd)
- end
- if use_verbose and name == "integration" then
- cmd = string.format("%s --verbose", cmd)
- end
- for _, arg in ipairs(extra_args) do
- cmd = string.format("%s %s", cmd, arg)
- end
-
- -- Tell TestListener where to write the summary
- local summary_file = path.join(summary_dir, target .. ".txt")
- os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file)
-
- -- Run test with real-time streaming output
- local test_ok = true
- try {
- function()
- os.exec(cmd)
- end,
- catch {
- function(errors)
- test_ok = false
- end
- }
- }
-
- -- Read summary written by TestListener
- local summary = parse_summary_file(summary_file)
- os.tryrm(summary_file)
-
- if not test_ok then
- any_failed = true
- end
-
- table.insert(results, {
- suite = suite_name,
- cases_passed = summary and summary.cases_passed or 0,
- cases_total = summary and summary.cases_total or 0,
- asserts_passed = summary and summary.asserts_passed or 0,
- asserts_total = summary and summary.asserts_total or 0,
- elapsed_seconds = summary and summary.elapsed_seconds or 0,
- failures = summary and summary.failures or {},
- passed = test_ok
- })
- end
-
- if any_failed then
- if repeat_count > 1 then
- printf("\n*** Failure detected on iteration %d, stopping ***\n", iteration)
- end
- break
- end
- end
-
- -- Clean up
- os.setenv("ZEN_TEST_SUMMARY_FILE", "")
- os.tryrm(summary_dir)
-
- -- Print JUnit reports if requested
- for test, junit_report_file in pairs(junit_report_files) do
- printf("=== report - %s ===\n", test)
- if os.isfile(junit_report_file) then
- local data = io.readfile(junit_report_file)
- if data then
- print(data)
- end
- end
- end
-
- -- Print summary table
- if #results > 0 then
- -- Calculate column widths based on content
- local col_suite = #("Suite")
- local col_cases = #("Cases")
- local col_asserts = #("Assertions")
- local col_time = #("Time")
- local col_status = #("Status")
-
- -- Compute totals
- local total_cases_passed = 0
- local total_cases_total = 0
- local total_asserts_passed = 0
- local total_asserts_total = 0
- local total_elapsed = 0
-
- for _, r in ipairs(results) do
- col_suite = math.max(col_suite, #r.suite)
- local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total)
- col_cases = math.max(col_cases, #cases_str)
- local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total)
- col_asserts = math.max(col_asserts, #asserts_str)
- col_time = math.max(col_time, #format_time(r.elapsed_seconds))
- local status_str = r.passed and "SUCCESS" or "FAILED"
- col_status = math.max(col_status, #status_str)
-
- total_cases_passed = total_cases_passed + r.cases_passed
- total_cases_total = total_cases_total + r.cases_total
- total_asserts_passed = total_asserts_passed + r.asserts_passed
- total_asserts_total = total_asserts_total + r.asserts_total
- total_elapsed = total_elapsed + r.elapsed_seconds
- end
-
- -- Account for totals row in column widths
- col_suite = math.max(col_suite, #("Total"))
- col_cases = math.max(col_cases, #(format_number(total_cases_passed) .. "/" .. format_number(total_cases_total)))
- col_asserts = math.max(col_asserts, #(format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total)))
- col_time = math.max(col_time, #format_time(total_elapsed))
-
- -- Add padding (1 space each side)
- col_suite = col_suite + 2
- col_cases = col_cases + 2
- col_asserts = col_asserts + 2
- col_time = col_time + 2
- col_status = col_status + 2
-
- -- Build horizontal border segments
- local h_suite = string.rep("-", col_suite)
- local h_cases = string.rep("-", col_cases)
- local h_asserts = string.rep("-", col_asserts)
- local h_time = string.rep("-", col_time)
- local h_status = string.rep("-", col_status)
-
- local top = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
- local mid = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
- local bottom = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+"
- local vbar = "|"
-
- local header_msg = any_failed and "Some tests failed:" or "All tests passed:"
- printf("\n* %s\n", header_msg)
- printf(" %s\n", top)
- printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, center_str("Suite", col_suite), vbar, center_str("Cases", col_cases), vbar, center_str("Assertions", col_asserts), vbar, center_str("Time", col_time), vbar, center_str("Status", col_status), vbar)
-
- for _, r in ipairs(results) do
- printf(" %s\n", mid)
- local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total)
- local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total)
- local time_str = format_time(r.elapsed_seconds)
- local status_str = r.passed and "SUCCESS" or "FAILED"
- printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str(r.suite, col_suite), vbar, right_align_str(cases_str, col_cases), vbar, right_align_str(asserts_str, col_asserts), vbar, right_align_str(time_str, col_time), vbar, right_align_str(status_str, col_status), vbar)
- end
-
- -- Totals row
- if #results > 1 then
- local h_suite_eq = string.rep("=", col_suite)
- local h_cases_eq = string.rep("=", col_cases)
- local h_asserts_eq = string.rep("=", col_asserts)
- local h_time_eq = string.rep("=", col_time)
- local h_status_eq = string.rep("=", col_status)
- local totals_sep = "+" .. h_suite_eq .. "+" .. h_cases_eq .. "+" .. h_asserts_eq .. "+" .. h_time_eq .. "+" .. h_status_eq .. "+"
- printf(" %s\n", totals_sep)
-
- local total_cases_str = format_number(total_cases_passed) .. "/" .. format_number(total_cases_total)
- local total_asserts_str = format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total)
- local total_time_str = format_time(total_elapsed)
- local total_status_str = any_failed and "FAILED" or "SUCCESS"
- printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str("Total", col_suite), vbar, right_align_str(total_cases_str, col_cases), vbar, right_align_str(total_asserts_str, col_asserts), vbar, right_align_str(total_time_str, col_time), vbar, right_align_str(total_status_str, col_status), vbar)
- end
-
- printf(" %s\n", bottom)
- end
-
- -- Print list of individual failing tests
- if any_failed then
- printf("\n Failures:\n")
- for _, r in ipairs(results) do
- if #r.failures > 0 then
- printf(" -- %s --\n", r.suite)
- for _, f in ipairs(r.failures) do
- printf(" FAILED: %s (%s:%d)\n", f.name, f.file, f.line)
- end
- elseif not r.passed then
- printf(" -- %s --\n", r.suite)
- printf(" (test binary exited with error, no failure details available)\n")
- end
- end
- end
-
- if any_failed then
- raise("one or more test suites failed")
- end
+ import("scripts.test")
+ test()
end)