diff options
| author | Stefan Boberg <[email protected]> | 2026-03-16 10:27:24 +0100 |
|---|---|---|
| committer | GitHub Enterprise <[email protected]> | 2026-03-16 10:27:24 +0100 |
| commit | 6df7bce35e84f91c868face688587c26a3765c7e (patch) | |
| tree | a2bf78a9b708707a6b2484c0a00c70abbc2f1891 | |
| parent | Add Docker image build for compute workers (#837) (diff) | |
| download | zen-6df7bce35e84f91c868face688587c26a3765c7e.tar.xz zen-6df7bce35e84f91c868face688587c26a3765c7e.zip | |
URI decoding, process env, compiler info, httpasio strands, regex route removal (#841)
- Percent-decode URIs in ASIO HTTP server to match http.sys CookedUrl behavior, ensuring consistent decoded paths across backends
- Add Environment field to CreateProcOptions for passing extra env vars to child processes (Windows: merged into Unicode environment block; Unix: setenv in fork)
- Add GetCompilerName() and include it in build options startup logging
- Suppress Windows CRT error dialogs in test harness for headless/CI runs
- Fix mimalloc package: pass CMAKE_BUILD_TYPE, skip cfuncs test for cross-compile
- Add virtual destructor to SentryAssertImpl to fix debug-mode warning
- Simplify object store path handling now that URIs arrive pre-decoded
- Add URI decoding test coverage for percent-encoded paths and query params
- Simplify httpasio request handling by using strands (guarantees no parallel handlers per connection)
- Removed deprecated regex-based route matching support
- Fix full GC never triggering after cross-toolchain builds: The `gc_state` file stores `system_clock` ticks, but the tick resolution differs between toolchains (nanoseconds on GCC/standard clang, microseconds on UE clang). A nanosecond timestamp misinterpreted as microseconds appears far in the future (~year 58,000), bypassing the staleness check and preventing time-based full GC from ever running. Fixed by also resetting when the stored timestamp is in the future.
- Clamp GC countdown display to configured interval: Prevents nonsensical log output (e.g. "Full GC in 492128002h") caused by the above or any other clock anomaly. The clamp applies to both the scheduler log and the status API.
| -rw-r--r-- | repo/packages/m/mimalloc/xmake.lua | 6 | ||||
| -rw-r--r-- | scripts/test.lua | 401 | ||||
| -rw-r--r-- | src/zencore/include/zencore/process.h | 9 | ||||
| -rw-r--r-- | src/zencore/include/zencore/system.h | 1 | ||||
| -rw-r--r-- | src/zencore/process.cpp | 52 | ||||
| -rw-r--r-- | src/zencore/sentryintegration.cpp | 2 | ||||
| -rw-r--r-- | src/zencore/system.cpp | 18 | ||||
| -rw-r--r-- | src/zencore/testing.cpp | 13 | ||||
| -rw-r--r-- | src/zenhttp/httpclient_test.cpp | 107 | ||||
| -rw-r--r-- | src/zenhttp/httpserver.cpp | 286 | ||||
| -rw-r--r-- | src/zenhttp/include/zenhttp/httpserver.h | 56 | ||||
| -rw-r--r-- | src/zenhttp/servers/httpasio.cpp | 180 | ||||
| -rw-r--r-- | src/zenserver/compute/computeserver.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver/hub/zenhubserver.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver/proxy/zenproxyserver.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver/storage/objectstore/objectstore.cpp | 11 | ||||
| -rw-r--r-- | src/zenserver/storage/zenstorageserver.cpp | 2 | ||||
| -rw-r--r-- | src/zenserver/zenserver.cpp | 9 | ||||
| -rw-r--r-- | src/zenstore/gc.cpp | 22 | ||||
| -rw-r--r-- | tsan.supp | 12 | ||||
| -rw-r--r-- | xmake.lua | 418 |
21 files changed, 829 insertions, 782 deletions
diff --git a/repo/packages/m/mimalloc/xmake.lua b/repo/packages/m/mimalloc/xmake.lua index 993e4c1a9..54d6613b8 100644 --- a/repo/packages/m/mimalloc/xmake.lua +++ b/repo/packages/m/mimalloc/xmake.lua @@ -35,7 +35,7 @@ package("mimalloc") end on_install("macosx", "windows", "linux", "android", "mingw", function (package) - local configs = {"-DMI_OVERRIDE=OFF"} + local configs = {"-DMI_OVERRIDE=OFF", "-DCMAKE_BUILD_TYPE=" .. (package:is_debug() and "Debug" or "Release")} table.insert(configs, "-DMI_BUILD_STATIC=" .. (package:config("shared") and "OFF" or "ON")) table.insert(configs, "-DMI_BUILD_SHARED=" .. (package:config("shared") and "ON" or "OFF")) table.insert(configs, "-DMI_SECURE=" .. (package:config("secure") and "ON" or "OFF")) @@ -71,5 +71,7 @@ package("mimalloc") end) on_test(function (package) - assert(package:has_cfuncs("mi_malloc", {includes = "mimalloc.h"})) + if not package:is_cross() then + assert(package:has_cfuncs("mi_malloc", {includes = "mimalloc.h"})) + end end) diff --git a/scripts/test.lua b/scripts/test.lua new file mode 100644 index 000000000..df1218ce8 --- /dev/null +++ b/scripts/test.lua @@ -0,0 +1,401 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +function main() + import("core.base.option") + import("core.project.config") + import("core.project.project") + + config.load() + + -- Override table: target name -> short name (for targets that don't follow convention) + local short_name_overrides = { + ["zenserver-test"] = "integration", + } + + -- Build test list from targets in the "tests" group + local available_tests = {} + for name, target in pairs(project.targets()) do + if target:get("group") == "tests" and name:endswith("-test") then + local short = short_name_overrides[name] + if not short then + -- Derive short name: "zencore-test" -> "core" + short = name + if short:startswith("zen") then short = short:sub(4) end + if short:endswith("-test") then short = short:sub(1, -6) end + end + table.insert(available_tests, {short, name}) + end + end + + -- Add non-test-group entries that have a test subcommand + table.insert(available_tests, {"server", "zenserver"}) + + table.sort(available_tests, function(a, b) return a[1] < b[1] end) + + -- Handle --list: print discovered test names and exit + if option.get("list") then + printf("Available tests:\n") + for _, entry in ipairs(available_tests) do + printf(" %-16s -> %s\n", entry[1], entry[2]) + end + return + end + + local testname = option.get("run") + + -- Parse comma-separated test names into a set + local requested = {} + for token in testname:gmatch("[^,]+") do + requested[token:match("^%s*(.-)%s*$")] = true + end + + -- Filter to requested test(s) + local tests = {} + local matched = {} + + for _, entry in ipairs(available_tests) do + local name, target = entry[1], entry[2] + if requested["all"] or requested[name] then + table.insert(tests, {name = name, target = target}) + matched[name] = true + end + end + + -- Check for unknown test names + if not requested["all"] then + for name, _ in pairs(requested) do + if not matched[name] then + raise("no tests match specification: '%s'", name) + end + end + end + + if #tests == 0 then + raise("no tests match specification: '%s'", testname) + end + + local plat, arch + if is_host("windows") then + plat = "windows" + arch = "x64" + elseif is_host("macosx") then + plat = "macosx" + arch = is_arch("arm64") and "arm64" or "x86_64" + else + plat = "linux" + arch = "x86_64" + end + + -- Only reconfigure if current config doesn't already match + if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then + local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or "" + local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or "" + os.exec("xmake config -y -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag) + end + + -- Build targets we're going to run + if requested["all"] then + os.exec("xmake build -y") + else + for _, entry in ipairs(tests) do + os.exec("xmake build -y %s", entry.target) + end + end + + local use_junit_reporting = option.get("junit") + local use_noskip = option.get("noskip") + local use_verbose = option.get("verbose") + local repeat_count = tonumber(option.get("repeat")) or 1 + local extra_args = option.get("arguments") or {} + local junit_report_files = {} + + local junit_report_dir + if use_junit_reporting then + junit_report_dir = path.join(os.projectdir(), config.get("buildir"), "reports") + os.mkdir(junit_report_dir) + end + + -- Results collection for summary table + local results = {} + local any_failed = false + + -- Format a number with thousands separators (e.g. 31103 -> "31,103") + local function format_number(n) + local s = tostring(n) + local pos = #s % 3 + if pos == 0 then pos = 3 end + local result = s:sub(1, pos) + for i = pos + 1, #s, 3 do + result = result .. "," .. s:sub(i, i + 2) + end + return result + end + + -- Center a string within a given width + local function center_str(s, width) + local pad = width - #s + local lpad = math.floor(pad / 2) + local rpad = pad - lpad + return string.rep(" ", lpad) .. s .. string.rep(" ", rpad) + end + + -- Left-align a string within a given width (with 1-space left margin) + local function left_align_str(s, width) + return " " .. s .. string.rep(" ", width - #s - 1) + end + + -- Right-align a string within a given width (with 1-space right margin) + local function right_align_str(s, width) + return string.rep(" ", width - #s - 1) .. s .. " " + end + + -- Format elapsed seconds as a human-readable string + local function format_time(seconds) + if seconds >= 60 then + local mins = math.floor(seconds / 60) + local secs = seconds - mins * 60 + return string.format("%dm %04.1fs", mins, secs) + else + return string.format("%.1fs", seconds) + end + end + + -- Parse test summary file written by TestListener + local function parse_summary_file(filepath) + if not os.isfile(filepath) then return nil end + local content = io.readfile(filepath) + if not content then return nil end + local ct = content:match("cases_total=(%d+)") + local cp = content:match("cases_passed=(%d+)") + local at = content:match("assertions_total=(%d+)") + local ap = content:match("assertions_passed=(%d+)") + if ct then + local failures = {} + for name, file, line in content:gmatch("failed=([^|\n]+)|([^|\n]+)|(%d+)") do + table.insert(failures, {name = name, file = file, line = tonumber(line)}) + end + local es = content:match("elapsed_seconds=([%d%.]+)") + return { + cases_total = tonumber(ct), + cases_passed = tonumber(cp) or 0, + asserts_total = tonumber(at) or 0, + asserts_passed = tonumber(ap) or 0, + elapsed_seconds = tonumber(es) or 0, + failures = failures + } + end + return nil + end + + -- Temp directory for summary files + local summary_dir = path.join(os.tmpdir(), "zen-test-summary") + os.mkdir(summary_dir) + + -- Run each test suite and collect results + for iteration = 1, repeat_count do + if repeat_count > 1 then + printf("\n*** Iteration %d/%d ***\n", iteration, repeat_count) + end + + for _, entry in ipairs(tests) do + local name, target = entry.name, entry.target + printf("=== %s ===\n", target) + + local suite_name = target + if name == "server" then + suite_name = "zenserver (test)" + end + + local cmd = string.format("xmake run %s", target) + if name == "server" then + cmd = string.format("xmake run %s test", target) + end + cmd = string.format("%s --duration=true", cmd) + + if use_junit_reporting then + local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target)) + junit_report_files[target] = junit_report_file + cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) + end + if use_noskip then + cmd = string.format("%s --no-skip", cmd) + end + if use_verbose and name == "integration" then + cmd = string.format("%s --verbose", cmd) + end + for _, arg in ipairs(extra_args) do + cmd = string.format("%s %s", cmd, arg) + end + + -- Tell TestListener where to write the summary + local summary_file = path.join(summary_dir, target .. ".txt") + os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file) + + -- Run test with real-time streaming output + local test_ok = true + try { + function() + os.exec(cmd) + end, + catch { + function(errors) + test_ok = false + end + } + } + + -- Read summary written by TestListener + local summary = parse_summary_file(summary_file) + os.tryrm(summary_file) + + if not test_ok then + any_failed = true + end + + table.insert(results, { + suite = suite_name, + cases_passed = summary and summary.cases_passed or 0, + cases_total = summary and summary.cases_total or 0, + asserts_passed = summary and summary.asserts_passed or 0, + asserts_total = summary and summary.asserts_total or 0, + elapsed_seconds = summary and summary.elapsed_seconds or 0, + failures = summary and summary.failures or {}, + passed = test_ok + }) + end + + if any_failed then + if repeat_count > 1 then + printf("\n*** Failure detected on iteration %d, stopping ***\n", iteration) + end + break + end + end + + -- Clean up + os.setenv("ZEN_TEST_SUMMARY_FILE", "") + os.tryrm(summary_dir) + + -- Print JUnit reports if requested + for test, junit_report_file in pairs(junit_report_files) do + printf("=== report - %s ===\n", test) + if os.isfile(junit_report_file) then + local data = io.readfile(junit_report_file) + if data then + print(data) + end + end + end + + -- Print summary table + if #results > 0 then + -- Calculate column widths based on content + local col_suite = #("Suite") + local col_cases = #("Cases") + local col_asserts = #("Assertions") + local col_time = #("Time") + local col_status = #("Status") + + -- Compute totals + local total_cases_passed = 0 + local total_cases_total = 0 + local total_asserts_passed = 0 + local total_asserts_total = 0 + local total_elapsed = 0 + + for _, r in ipairs(results) do + col_suite = math.max(col_suite, #r.suite) + local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) + col_cases = math.max(col_cases, #cases_str) + local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) + col_asserts = math.max(col_asserts, #asserts_str) + col_time = math.max(col_time, #format_time(r.elapsed_seconds)) + local status_str = r.passed and "SUCCESS" or "FAILED" + col_status = math.max(col_status, #status_str) + + total_cases_passed = total_cases_passed + r.cases_passed + total_cases_total = total_cases_total + r.cases_total + total_asserts_passed = total_asserts_passed + r.asserts_passed + total_asserts_total = total_asserts_total + r.asserts_total + total_elapsed = total_elapsed + r.elapsed_seconds + end + + -- Account for totals row in column widths + col_suite = math.max(col_suite, #("Total")) + col_cases = math.max(col_cases, #(format_number(total_cases_passed) .. "/" .. format_number(total_cases_total))) + col_asserts = math.max(col_asserts, #(format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total))) + col_time = math.max(col_time, #format_time(total_elapsed)) + + -- Add padding (1 space each side) + col_suite = col_suite + 2 + col_cases = col_cases + 2 + col_asserts = col_asserts + 2 + col_time = col_time + 2 + col_status = col_status + 2 + + -- Build horizontal border segments + local h_suite = string.rep("-", col_suite) + local h_cases = string.rep("-", col_cases) + local h_asserts = string.rep("-", col_asserts) + local h_time = string.rep("-", col_time) + local h_status = string.rep("-", col_status) + + local top = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local mid = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local bottom = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local vbar = "|" + + local header_msg = any_failed and "Some tests failed:" or "All tests passed:" + printf("\n* %s\n", header_msg) + printf(" %s\n", top) + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, center_str("Suite", col_suite), vbar, center_str("Cases", col_cases), vbar, center_str("Assertions", col_asserts), vbar, center_str("Time", col_time), vbar, center_str("Status", col_status), vbar) + + for _, r in ipairs(results) do + printf(" %s\n", mid) + local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) + local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) + local time_str = format_time(r.elapsed_seconds) + local status_str = r.passed and "SUCCESS" or "FAILED" + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str(r.suite, col_suite), vbar, right_align_str(cases_str, col_cases), vbar, right_align_str(asserts_str, col_asserts), vbar, right_align_str(time_str, col_time), vbar, right_align_str(status_str, col_status), vbar) + end + + -- Totals row + if #results > 1 then + local h_suite_eq = string.rep("=", col_suite) + local h_cases_eq = string.rep("=", col_cases) + local h_asserts_eq = string.rep("=", col_asserts) + local h_time_eq = string.rep("=", col_time) + local h_status_eq = string.rep("=", col_status) + local totals_sep = "+" .. h_suite_eq .. "+" .. h_cases_eq .. "+" .. h_asserts_eq .. "+" .. h_time_eq .. "+" .. h_status_eq .. "+" + printf(" %s\n", totals_sep) + + local total_cases_str = format_number(total_cases_passed) .. "/" .. format_number(total_cases_total) + local total_asserts_str = format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total) + local total_time_str = format_time(total_elapsed) + local total_status_str = any_failed and "FAILED" or "SUCCESS" + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str("Total", col_suite), vbar, right_align_str(total_cases_str, col_cases), vbar, right_align_str(total_asserts_str, col_asserts), vbar, right_align_str(total_time_str, col_time), vbar, right_align_str(total_status_str, col_status), vbar) + end + + printf(" %s\n", bottom) + end + + -- Print list of individual failing tests + if any_failed then + printf("\n Failures:\n") + for _, r in ipairs(results) do + if #r.failures > 0 then + printf(" -- %s --\n", r.suite) + for _, f in ipairs(r.failures) do + printf(" FAILED: %s (%s:%d)\n", f.name, f.file, f.line) + end + elseif not r.passed then + printf(" -- %s --\n", r.suite) + printf(" (test binary exited with error, no failure details available)\n") + end + end + end + + if any_failed then + raise("one or more test suites failed") + end +end diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index 809312c7b..3177f64c1 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -6,6 +6,9 @@ #include <zencore/zencore.h> #include <filesystem> +#include <string> +#include <utility> +#include <vector> namespace zen { @@ -68,6 +71,12 @@ struct CreateProcOptions const std::filesystem::path* WorkingDirectory = nullptr; uint32_t Flags = 0; std::filesystem::path StdoutFile; + + /// Additional environment variables for the child process. These are merged + /// with the parent's environment — existing variables are inherited, and + /// entries here override or add to them. + std::vector<std::pair<std::string, std::string>> Environment; + #if ZEN_PLATFORM_WINDOWS JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed #endif diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index a67999e52..2e39cc660 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -17,6 +17,7 @@ std::string_view GetOperatingSystemName(); std::string GetOperatingSystemVersion(); std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); +std::string_view GetCompilerName(); struct SystemMetrics { diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index f657869dc..852678ffe 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -11,6 +11,7 @@ #include <zencore/timer.h> #include <zencore/trace.h> +#include <map> #include <thread> ZEN_THIRD_PARTY_INCLUDES_START @@ -487,13 +488,57 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma STARTUPINFO StartupInfo{.cb = sizeof(STARTUPINFO)}; bool InheritHandles = false; - void* Environment = nullptr; LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; + // Build environment block when custom environment variables are specified + ExtendableWideStringBuilder<512> EnvironmentBlock; + void* Environment = nullptr; + if (!Options.Environment.empty()) + { + // Capture current environment into a map + std::map<std::wstring, std::wstring> EnvMap; + wchar_t* EnvStrings = GetEnvironmentStringsW(); + if (EnvStrings) + { + for (const wchar_t* Ptr = EnvStrings; *Ptr; Ptr += wcslen(Ptr) + 1) + { + std::wstring_view Entry(Ptr); + size_t EqPos = Entry.find(L'='); + if (EqPos != std::wstring_view::npos && EqPos > 0) + { + EnvMap[std::wstring(Entry.substr(0, EqPos))] = std::wstring(Entry.substr(EqPos + 1)); + } + } + FreeEnvironmentStringsW(EnvStrings); + } + + // Apply overrides + for (const auto& [Key, Value] : Options.Environment) + { + EnvMap[Utf8ToWide(Key)] = Utf8ToWide(Value); + } + + // Build double-null-terminated environment block + for (const auto& [Key, Value] : EnvMap) + { + EnvironmentBlock << Key; + EnvironmentBlock.Append(L'='); + EnvironmentBlock << Value; + EnvironmentBlock.Append(L'\0'); + } + EnvironmentBlock.Append(L'\0'); + + Environment = EnvironmentBlock.Data(); + } + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); DWORD CreationFlags = 0; + if (Environment) + { + CreationFlags |= CREATE_UNICODE_ENVIRONMENT; + } if (Options.Flags & CreateProcOptions::Flag_NewConsole) { CreationFlags |= CREATE_NEW_CONSOLE; @@ -790,6 +835,11 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine } } + for (const auto& [Key, Value] : Options.Environment) + { + setenv(Key.c_str(), Value.c_str(), 1); + } + if (execv(Executable.c_str(), ArgV.data()) < 0) { ThrowLastError("Failed to exec() a new process image"); diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 58b76783a..b7d01003b 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -31,6 +31,8 @@ namespace { struct SentryAssertImpl : zen::AssertImpl { + ZEN_DEBUG_SECTION ~SentryAssertImpl() override = default; + virtual void ZEN_FORCENOINLINE ZEN_DEBUG_SECTION OnAssert(const char* Filename, int LineNumber, const char* FunctionName, diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 141450b84..8985a8a76 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -660,6 +660,24 @@ GetCpuName() #endif } +std::string_view +GetCompilerName() +{ +#define ZEN_STRINGIFY_IMPL(x) #x +#define ZEN_STRINGIFY(x) ZEN_STRINGIFY_IMPL(x) +#if ZEN_COMPILER_CLANG + return "clang " ZEN_STRINGIFY(__clang_major__) "." ZEN_STRINGIFY(__clang_minor__) "." ZEN_STRINGIFY(__clang_patchlevel__); +#elif ZEN_COMPILER_MSC + return "MSVC " ZEN_STRINGIFY(_MSC_VER); +#elif ZEN_COMPILER_GCC + return "GCC " ZEN_STRINGIFY(__GNUC__) "." ZEN_STRINGIFY(__GNUC_MINOR__) "." ZEN_STRINGIFY(__GNUC_PATCHLEVEL__); +#else + return "unknown"; +#endif +#undef ZEN_STRINGIFY +#undef ZEN_STRINGIFY_IMPL +} + void Describe(const SystemMetrics& Metrics, CbWriter& Writer) { diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index d7eb3b17d..f5bc723b1 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -24,6 +24,8 @@ # if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include <execinfo.h> # include <unistd.h> +# elif ZEN_PLATFORM_WINDOWS +# include <crtdbg.h> # endif namespace zen::testing { @@ -296,6 +298,17 @@ RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink } # endif +# if ZEN_PLATFORM_WINDOWS + // Suppress Windows error dialogs (crash/abort/assert) so tests terminate + // immediately instead of blocking on a modal dialog in CI or headless runs. + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + _set_abort_behavior(0, _WRITE_ABORT_MSG); + _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDERR); + _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_FILE); + _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDERR); +# endif + zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); InstallCrashSignalHandlers(); diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp index 5f3ad2455..3ca586f87 100644 --- a/src/zenhttp/httpclient_test.cpp +++ b/src/zenhttp/httpclient_test.cpp @@ -154,6 +154,42 @@ public: }, HttpVerb::kGet); + m_Router.AddMatcher("anypath", [](std::string_view Str) -> bool { return !Str.empty(); }); + + m_Router.RegisterRoute( + "echo/uri", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string Body = std::string(HttpReq.RelativeUri()); + + auto Params = HttpReq.GetQueryParams(); + for (const auto& [Key, Value] : Params.KvPairs) + { + Body += fmt::format("\n{}={}", Key, Value); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body); + }, + HttpVerb::kGet | HttpVerb::kPut); + + m_Router.RegisterRoute( + "echo/uri/{anypath}", + [](HttpRouterRequest& Req) { + // Echo both the RelativeUri and the captured path segment + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view Captured = Req.GetCapture(1); + std::string Body = fmt::format("uri={}\ncapture={}", HttpReq.RelativeUri(), Captured); + + auto Params = HttpReq.GetQueryParams(); + for (const auto& [Key, Value] : Params.KvPairs) + { + Body += fmt::format("\n{}={}", Key, Value); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Body); + }, + HttpVerb::kGet | HttpVerb::kPut); + m_Router.RegisterRoute( "slow", [](HttpRouterRequest& Req) { @@ -1689,6 +1725,77 @@ TEST_CASE("httpclient.https") # endif // ZEN_USE_OPENSSL +TEST_CASE("httpclient.uri_decoding") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + // URI without encoding — should pass through unchanged + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello/world.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/hello/world.txt\ncapture=hello/world.txt"); + } + + // Percent-encoded space — server should see decoded path + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt"); + } + + // Percent-encoded slash (%2F) — should be decoded to / + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/a%2Fb.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/a/b.txt\ncapture=a/b.txt"); + } + + // Multiple encodings in one path + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/file%20%26%20name.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/file & name.txt\ncapture=file & name.txt"); + } + + // No capture — echo/uri route returns just RelativeUri + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "echo/uri"); + } + + // Literal percent that is not an escape (%ZZ) — should be kept as-is + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/100%25done.txt"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "uri=echo/uri/100%done.txt\ncapture=100%done.txt"); + } + + // Query params — raw values are returned as-is from GetQueryParams + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri?key=value&name=test"); + REQUIRE(Resp.IsSuccess()); + CHECK(Resp.AsText() == "echo/uri\nkey=value\nname=test"); + } + + // Query params with percent-encoded values + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri?prefix=listing%2F&mode=s3"); + REQUIRE(Resp.IsSuccess()); + // GetQueryParams returns raw (still-encoded) values — callers must Decode() explicitly + CHECK(Resp.AsText() == "echo/uri\nprefix=listing%2F\nmode=s3"); + } + + // Query params with path capture and encoding + { + HttpClient::Response Resp = Client.Get("/api/test/echo/uri/hello%20world.txt?tag=a%26b"); + REQUIRE(Resp.IsSuccess()); + // Path is decoded, query values are raw + CHECK(Resp.AsText() == "uri=echo/uri/hello world.txt\ncapture=hello world.txt\ntag=a%26b"); + } +} + TEST_SUITE_END(); void diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 672467f56..4d98e9650 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -700,15 +700,6 @@ HttpServerRequest::ReadPayloadPackage() ////////////////////////////////////////////////////////////////////////// void -HttpRequestRouter::AddPattern(const char* Id, const char* Regex) -{ - ZEN_ASSERT(m_PatternMap.find(Id) == m_PatternMap.end()); - ZEN_ASSERT(!m_IsFinalized); - - m_PatternMap.insert({Id, Regex}); -} - -void HttpRequestRouter::AddMatcher(const char* Id, std::function<bool(std::string_view)>&& Matcher) { ZEN_ASSERT(m_MatcherNameMap.find(Id) == m_MatcherNameMap.end()); @@ -724,170 +715,77 @@ HttpRequestRouter::RegisterRoute(const char* UriPattern, HttpRequestRouter::Hand { ZEN_ASSERT(!m_IsFinalized); - if (ExtendableStringBuilder<128> ExpandedRegex; ProcessRegexSubstitutions(UriPattern, ExpandedRegex)) - { - // Regex route - m_RegexHandlers.emplace_back(ExpandedRegex.c_str(), SupportedVerbs, std::move(HandlerFunc), UriPattern); - } - else - { - // New-style regex-free route. More efficient and should be used for everything eventually + int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern)); - int RegexLen = gsl::narrow_cast<int>(strlen(UriPattern)); + int i = 0; - int i = 0; + std::vector<int> MatcherIndices; - std::vector<int> MatcherIndices; - - while (i < RegexLen) + while (i < RegexLen) + { + if (UriPattern[i] == '{') { - if (UriPattern[i] == '{') + bool IsComplete = false; + int PatternStart = i + 1; + while (++i < RegexLen) { - bool IsComplete = false; - int PatternStart = i + 1; - while (++i < RegexLen) + if (UriPattern[i] == '}') { - if (UriPattern[i] == '}') + if (i == PatternStart) { - if (i == PatternStart) - { - throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern)); - } - std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart); - if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end()) - { - // It's a match - MatcherIndices.push_back(it->second); - IsComplete = true; - ++i; - break; - } - else - { - throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern)); - } + throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern)); } - } - if (!IsComplete) - { - throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern)); - } - } - else - { - if (UriPattern[i] == '/') - { - throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern)); - } - - int SegmentStart = i; - while (++i < RegexLen && UriPattern[i] != '/') - ; - - std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart)); - int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size()); - m_Literals.push_back(std::string(Segment)); - MatcherIndices.push_back(-1 - LiteralIndex); - } - - if (i < RegexLen && UriPattern[i] == '/') - { - ++i; // skip slash - } - } - - m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern); - } -} - -std::string_view -HttpRouterRequest::GetCapture(uint32_t Index) const -{ - if (!m_CapturedSegments.empty()) - { - ZEN_ASSERT(Index < m_CapturedSegments.size()); - return m_CapturedSegments[Index]; - } - - ZEN_ASSERT(Index < m_Match.size()); - - const auto& Match = m_Match[Index]; - - return std::string_view(&*Match.first, Match.second - Match.first); -} - -bool -HttpRequestRouter::ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& OutExpandedRegex) -{ - size_t RegexLen = strlen(Regex); - - bool HasRegex = false; - - std::vector<std::string> UnknownPatterns; - - for (size_t i = 0; i < RegexLen;) - { - bool matched = false; - - if (Regex[i] == '{' && ((i == 0) || (Regex[i - 1] != '\\'))) - { - // Might have a pattern reference - find closing brace - - for (size_t j = i + 1; j < RegexLen; ++j) - { - if (Regex[j] == '}') - { - std::string Pattern(&Regex[i + 1], j - i - 1); - - if (auto it = m_PatternMap.find(Pattern); it != m_PatternMap.end()) + std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart); + if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end()) { - OutExpandedRegex.Append(it->second.c_str()); - HasRegex = true; + // It's a match + MatcherIndices.push_back(it->second); + IsComplete = true; + ++i; + break; } else { - UnknownPatterns.push_back(Pattern); + throw std::runtime_error(fmt::format("unknown matcher pattern '{}' in URI pattern '{}'", Pattern, UriPattern)); } - - // skip ahead - i = j + 1; - - matched = true; - - break; } } + if (!IsComplete) + { + throw std::runtime_error(fmt::format("unterminated matcher pattern in URI pattern '{}'", UriPattern)); + } } - - if (!matched) - { - OutExpandedRegex.Append(Regex[i++]); - } - } - - if (HasRegex) - { - if (UnknownPatterns.size() > 0) + else { - std::string UnknownList; - for (const auto& Pattern : UnknownPatterns) + if (UriPattern[i] == '/') { - if (!UnknownList.empty()) - { - UnknownList += ", "; - } - UnknownList += "'"; - UnknownList += Pattern; - UnknownList += "'"; + throw std::runtime_error(fmt::format("unexpected '/' in literal segment of URI pattern '{}'", UriPattern)); } - throw std::runtime_error(fmt::format("unknown pattern(s) {} in regex route '{}'", UnknownList, Regex)); + int SegmentStart = i; + while (++i < RegexLen && UriPattern[i] != '/') + ; + + std::string_view Segment(&UriPattern[SegmentStart], (i - SegmentStart)); + int LiteralIndex = gsl::narrow_cast<int>(m_Literals.size()); + m_Literals.push_back(std::string(Segment)); + MatcherIndices.push_back(-1 - LiteralIndex); } - return true; + if (i < RegexLen && UriPattern[i] == '/') + { + ++i; // skip slash + } } - return false; + m_MatcherEndpoints.emplace_back(std::move(MatcherIndices), SupportedVerbs, std::move(HandlerFunc), UriPattern); +} + +std::string_view +HttpRouterRequest::GetCapture(uint32_t Index) const +{ + ZEN_ASSERT(Index < m_CapturedSegments.size()); + return m_CapturedSegments[Index]; } bool @@ -903,8 +801,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) std::string_view Uri = Request.RelativeUri(); HttpRouterRequest RouterRequest(Request); - // First try new-style matcher routes - for (const MatcherEndpoint& Handler : m_MatcherEndpoints) { if ((Handler.Verbs & Verb) == Verb) @@ -1002,28 +898,6 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) } } - // Old-style regex routes - - for (const auto& Handler : m_RegexHandlers) - { - if ((Handler.Verbs & Verb) == Verb && regex_match(begin(Uri), end(Uri), RouterRequest.m_Match, Handler.RegEx)) - { -#if ZEN_WITH_OTEL - if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan()) - { - ExtendableStringBuilder<128> RoutePath; - RoutePath.Append(Request.Service().BaseUri()); - RoutePath.Append(Handler.Pattern); - ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView()); - } -#endif - - Handler.Handler(RouterRequest); - - return true; // Route matched - } - } - return false; // No route matched } @@ -1422,72 +1296,6 @@ TEST_CASE("http.common") virtual uint32_t ParseRequestId() const override { return 0; } }; - SUBCASE("router-regex") - { - bool HandledA = false; - bool HandledAA = false; - std::vector<std::string> Captures; - auto Reset = [&] { - Captures.clear(); - HandledA = HandledAA = false; - }; - - TestHttpService Service; - - HttpRequestRouter r; - r.AddPattern("a", "([[:alpha:]]+)"); - r.RegisterRoute( - "{a}", - [&](auto& Req) { - HandledA = true; - Captures = {std::string(Req.GetCapture(0))}; - }, - HttpVerb::kGet); - - r.RegisterRoute( - "{a}/{a}", - [&](auto& Req) { - HandledAA = true; - Captures = {std::string(Req.GetCapture(1)), std::string(Req.GetCapture(2))}; - }, - HttpVerb::kGet); - - { - Reset(); - TestHttpServerRequest req(Service, "abc"sv); - r.HandleRequest(req); - CHECK(HandledA); - CHECK(!HandledAA); - REQUIRE_EQ(Captures.size(), 1); - CHECK_EQ(Captures[0], "abc"sv); - } - - { - Reset(); - TestHttpServerRequest req{Service, "abc/def"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - CHECK(HandledAA); - REQUIRE_EQ(Captures.size(), 2); - CHECK_EQ(Captures[0], "abc"sv); - CHECK_EQ(Captures[1], "def"sv); - } - - { - Reset(); - TestHttpServerRequest req{Service, "123"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - } - - { - Reset(); - TestHttpServerRequest req{Service, "a123"sv}; - r.HandleRequest(req); - CHECK(!HandledA); - } - } - SUBCASE("router-matcher") { bool HandledA = false; diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 77feb6568..2a8b2ca94 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -20,7 +20,6 @@ #include <gsl/gsl-lite.hpp> #include <list> #include <map> -#include <regex> #include <span> #include <unordered_map> @@ -357,9 +356,8 @@ class HttpRouterRequest public: /** Get captured segment from matched URL * - * @param Index Index of captured segment to retrieve. Note that due to - * backwards compatibility with regex-based routes, this index is 1-based - * and index=0 is the full matched URL + * @param Index Index of captured segment to retrieve. Index 0 is the full + * matched URL, subsequent indices are the matched segments in order. * @return Returns string view of captured segment */ std::string_view GetCapture(uint32_t Index) const; @@ -372,11 +370,8 @@ private: HttpRouterRequest(const HttpRouterRequest&) = delete; HttpRouterRequest& operator=(const HttpRouterRequest&) = delete; - using MatchResults_t = std::match_results<std::string_view::const_iterator>; - HttpServerRequest& m_HttpRequest; - MatchResults_t m_Match; - std::vector<std::string_view> m_CapturedSegments; // for matcher-based routes + std::vector<std::string_view> m_CapturedSegments; friend class HttpRequestRouter; }; @@ -384,9 +379,7 @@ private: /** HTTP request router helper * * This helper class allows a service implementer to register one or more - * endpoints using pattern matching. We currently support a legacy regex-based - * matching system, but also a new matcher-function based system which is more - * efficient and should be used whenever possible. + * endpoints using pattern matching with matcher functions. * * This is intended to be initialized once only, there is no thread * safety so you can absolutely not add or remove endpoints once the handler @@ -405,13 +398,6 @@ public: typedef std::function<void(HttpRouterRequest&)> HandlerFunc_t; /** - * @brief Add pattern which can be referenced by name, commonly used for URL components - * @param Id String used to identify patterns for replacement - * @param Regex String which will replace the Id string in any registered URL paths - */ - void AddPattern(const char* Id, const char* Regex); - - /** * @brief Add matcher function which can be referenced by name, used for URL components * @param Id String used to identify matchers in endpoint specifications * @param Matcher Function which will be called to match the component @@ -421,8 +407,8 @@ public: /** * @brief Register an endpoint handler for the given route * @param Pattern Pattern used to match the handler to a request. This should - * only contain literal URI segments and pattern aliases registered - via AddPattern() or AddMatcher() + * only contain literal URI segments and matcher aliases registered + via AddMatcher() * @param HandlerFunc Handler function to call for any matching request * @param SupportedVerbs Supported HTTP verbs for this handler */ @@ -437,36 +423,6 @@ public: bool HandleRequest(zen::HttpServerRequest& Request); private: - bool ProcessRegexSubstitutions(const char* Regex, StringBuilderBase& ExpandedRegex); - - struct RegexEndpoint - { - RegexEndpoint(const char* Regex, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) - : RegEx(Regex, std::regex::icase | std::regex::ECMAScript) - , Verbs(SupportedVerbs) - , Handler(std::move(Handler)) - , Pattern(Pattern) - { - } - - ~RegexEndpoint() = default; - - std::regex RegEx; - HttpVerb Verbs; - HandlerFunc_t Handler; - const char* Pattern; - - private: - RegexEndpoint& operator=(const RegexEndpoint&) = delete; - RegexEndpoint(const RegexEndpoint&) = delete; - }; - - std::list<RegexEndpoint> m_RegexHandlers; - std::unordered_map<std::string, std::string> m_PatternMap; - - // New-style matcher endpoints. Should be preferred over regex endpoints where possible - // as it is considerably more efficient - struct MatcherEndpoint { MatcherEndpoint(std::vector<int>&& ComponentIndices, HttpVerb SupportedVerbs, HandlerFunc_t&& Handler, const char* Pattern) diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 643f33618..9f4875eaf 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -601,6 +601,7 @@ public: bool m_IsLocalMachineRequest; bool m_AllowZeroCopyFileSend = true; std::string m_RemoteAddress; + std::string m_DecodedUri; // Percent-decoded URI; m_Uri/m_UriWithExtension point into this std::unique_ptr<HttpResponse> m_Response; }; @@ -623,6 +624,7 @@ public: ~HttpResponse() = default; void SetAllowZeroCopyFileSend(bool Allow) { m_AllowZeroCopyFileSend = Allow; } + void SetKeepAlive(bool KeepAlive) { m_IsKeepAlive = KeepAlive; } /** * Initialize the response for sending a payload made up of multiple blobs @@ -780,8 +782,8 @@ public: return m_Headers; } - template<typename SocketType> - void SendResponse(SocketType& Socket, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token) + template<typename SocketType, typename Executor> + void SendResponse(SocketType& Socket, Executor& Strand, std::function<void(const asio::error_code& Ec, std::size_t ByteCount)>&& Token) { ZEN_ASSERT(m_State == State::kInitialized); @@ -791,11 +793,11 @@ public: m_SendCb = std::move(Token); m_State = State::kSending; - SendNextChunk(Socket); + SendNextChunk(Socket, Strand); } - template<typename SocketType> - void SendNextChunk(SocketType& Socket) + template<typename SocketType, typename Executor> + void SendNextChunk(SocketType& Socket, Executor& Strand) { ZEN_ASSERT(m_State == State::kSending); @@ -812,12 +814,12 @@ public: auto CompletionToken = [Self = this, Token = std::move(m_SendCb), TotalBytes = m_TotalBytesSent] { Token({}, TotalBytes); }; - asio::defer(Socket.get_executor(), std::move(CompletionToken)); + asio::defer(Strand, std::move(CompletionToken)); return; } - auto OnCompletion = [this, &Socket](const asio::error_code& Ec, std::size_t ByteCount) { + auto OnCompletion = asio::bind_executor(Strand, [this, &Socket, &Strand](const asio::error_code& Ec, std::size_t ByteCount) { ZEN_ASSERT(m_State == State::kSending); m_TotalBytesSent += ByteCount; @@ -828,9 +830,9 @@ public: } else { - SendNextChunk(Socket); + SendNextChunk(Socket, Strand); } - }; + }); const IoVec& Io = m_IoVecs[m_IoVecCursor++]; @@ -982,16 +984,14 @@ private: void CloseConnection(); void SendInlineResponse(uint32_t RequestNumber, std::string_view StatusLine, std::string_view Headers = {}, std::string_view Body = {}); - HttpAsioServerImpl& m_Server; - asio::streambuf m_RequestBuffer; - std::atomic<uint32_t> m_RequestCounter{0}; - uint32_t m_ConnectionId = 0; - Ref<IHttpPackageHandler> m_PackageHandler; - - RwLock m_ActiveResponsesLock; + HttpAsioServerImpl& m_Server; + std::unique_ptr<SocketType> m_Socket; + asio::strand<asio::any_io_executor> m_Strand; + asio::streambuf m_RequestBuffer; + uint32_t m_RequestCounter = 0; + uint32_t m_ConnectionId = 0; + Ref<IHttpPackageHandler> m_PackageHandler; std::deque<std::unique_ptr<HttpResponse>> m_ActiveResponses; - - std::unique_ptr<SocketType> m_Socket; }; std::atomic<uint32_t> g_ConnectionIdCounter{0}; @@ -999,8 +999,9 @@ std::atomic<uint32_t> g_ConnectionIdCounter{0}; template<typename SocketType> HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Server, std::unique_ptr<SocketType>&& Socket) : m_Server(Server) -, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1)) , m_Socket(std::move(Socket)) +, m_Strand(asio::make_strand(m_Socket->get_executor())) +, m_ConnectionId(g_ConnectionIdCounter.fetch_add(1)) { ZEN_TRACE_VERBOSE("new connection #{}", m_ConnectionId); } @@ -1008,8 +1009,6 @@ HttpServerConnectionT<SocketType>::HttpServerConnectionT(HttpAsioServerImpl& Ser template<typename SocketType> HttpServerConnectionT<SocketType>::~HttpServerConnectionT() { - RwLock::ExclusiveLockScope _(m_ActiveResponsesLock); - ZEN_TRACE_VERBOSE("destroying connection #{}", m_ConnectionId); } @@ -1017,7 +1016,7 @@ template<typename SocketType> void HttpServerConnectionT<SocketType>::HandleNewRequest() { - EnqueueRead(); + asio::dispatch(m_Strand, [Conn = AsSharedPtr()] { Conn->EnqueueRead(); }); } template<typename SocketType> @@ -1058,7 +1057,9 @@ HttpServerConnectionT<SocketType>::EnqueueRead() asio::async_read(*m_Socket.get(), m_RequestBuffer, asio::transfer_at_least(1), - [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnDataReceived(Ec, ByteCount); }); + asio::bind_executor(m_Strand, [Conn = AsSharedPtr()](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnDataReceived(Ec, ByteCount); + })); } template<typename SocketType> @@ -1091,7 +1092,7 @@ HttpServerConnectionT<SocketType>::OnDataReceived(const asio::error_code& Ec, [[ ZEN_TRACE_VERBOSE("on data received, connection: {}, request: {}, thread: {}, bytes: {}", m_ConnectionId, - m_RequestCounter.load(std::memory_order_relaxed), + m_RequestCounter, zen::GetCurrentThreadId(), NiceBytes(ByteCount)); @@ -1153,25 +1154,23 @@ HttpServerConnectionT<SocketType>::OnResponseDataSent(const asio::error_code& if (ResponseToPop) { - m_ActiveResponsesLock.WithExclusiveLock([&] { - // Once a response is sent we can release any referenced resources - // - // completion callbacks may be issued out-of-order so we need to - // remove the relevant entry from our active response list, it may - // not be the first - - if (auto It = find_if(begin(m_ActiveResponses), - end(m_ActiveResponses), - [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; }); - It != end(m_ActiveResponses)) - { - m_ActiveResponses.erase(It); - } - else - { - ZEN_WARN("response not found"); - } - }); + // Once a response is sent we can release any referenced resources + // + // completion callbacks may be issued out-of-order so we need to + // remove the relevant entry from our active response list, it may + // not be the first + + if (auto It = find_if(begin(m_ActiveResponses), + end(m_ActiveResponses), + [ResponseToPop](const auto& Item) { return Item.get() == ResponseToPop; }); + It != end(m_ActiveResponses)) + { + m_ActiveResponses.erase(It); + } + else + { + ZEN_WARN("response not found"); + } } if (!m_RequestData.IsKeepAlive()) @@ -1234,9 +1233,11 @@ HttpServerConnectionT<SocketType>::SendInlineResponse(uint32_t RequestNumber asio::async_write( *m_Socket, Buffer, - [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + })); } template<typename SocketType> @@ -1272,21 +1273,23 @@ HttpServerConnectionT<SocketType>::HandleRequest() asio::async_write( *m_Socket, asio::buffer(ResponseStr->data(), ResponseStr->size()), - [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) { - if (Ec) - { - ZEN_WARN("WebSocket 101 send failed: {}", Ec.message()); - return; - } - - Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened(); - using WsConnType = WsAsioConnectionT<SocketType>; - Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer)); - Ref<WebSocketConnection> WsConnRef(WsConn.Get()); - - WsHandler->OnWebSocketOpen(std::move(WsConnRef)); - WsConn->Start(); - }); + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) { + if (Ec) + { + ZEN_WARN("WebSocket 101 send failed: {}", Ec.message()); + return; + } + + Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened(); + using WsConnType = WsAsioConnectionT<SocketType>; + Ref<WsConnType> WsConn(new WsConnType(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer)); + Ref<WebSocketConnection> WsConnRef(WsConn.Get()); + + WsHandler->OnWebSocketOpen(std::move(WsConnRef)); + WsConn->Start(); + })); m_RequestState = RequestState::kDone; return; @@ -1312,7 +1315,7 @@ HttpServerConnectionT<SocketType>::HandleRequest() m_RequestState = RequestState::kWriting; } - const uint32_t RequestNumber = m_RequestCounter.fetch_add(1); + const uint32_t RequestNumber = m_RequestCounter++; if (HttpService* Service = m_Server.RouteRequest(m_RequestData.Url())) { @@ -1444,31 +1447,34 @@ HttpServerConnectionT<SocketType>::HandleRequest() { ZEN_TRACE_CPU("asio::async_write"); - std::string_view Headers = Response->GetHeaders(); + HttpResponse* ResponseRaw = Response.get(); + m_ActiveResponses.push_back(std::move(Response)); + + std::string_view Headers = ResponseRaw->GetHeaders(); std::vector<asio::const_buffer> AsioBuffers; AsioBuffers.push_back(asio::const_buffer(Headers.data(), Headers.size())); - asio::async_write(*m_Socket.get(), - AsioBuffers, - asio::transfer_all(), - [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + asio::async_write( + *m_Socket.get(), + AsioBuffers, + asio::transfer_all(), + asio::bind_executor( + m_Strand, + [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw); + })); } else { ZEN_TRACE_CPU("asio::async_write"); HttpResponse* ResponseRaw = Response.get(); - - m_ActiveResponsesLock.WithExclusiveLock([&] { - // Keep referenced resources alive - m_ActiveResponses.push_back(std::move(Response)); - }); + m_ActiveResponses.push_back(std::move(Response)); ResponseRaw->SendResponse( *m_Socket, + m_Strand, [Conn = AsSharedPtr(), ResponseRaw, RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ ResponseRaw); }); @@ -1982,11 +1988,24 @@ HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, { const int PrefixLength = Service.UriPrefixLength(); - std::string_view Uri = Request.Url(); - Uri.remove_prefix(std::min(PrefixLength, static_cast<int>(Uri.size()))); - m_Uri = Uri; - m_UriWithExtension = Uri; - m_QueryString = Request.QueryString(); + std::string_view RawUri = Request.Url(); + RawUri.remove_prefix(std::min(PrefixLength, static_cast<int>(RawUri.size()))); + + // Percent-decode the URI path so handlers see the same decoded paths regardless + // of whether the ASIO or http.sys backend is used (http.sys pre-decodes via CookedUrl). + // Skip the allocation when there is nothing to decode (common case). + if (RawUri.find('%') != std::string_view::npos) + { + m_DecodedUri = Decode(RawUri); + m_Uri = m_DecodedUri; + m_UriWithExtension = m_DecodedUri; + } + else + { + m_Uri = RawUri; + m_UriWithExtension = RawUri; + } + m_QueryString = Request.QueryString(); m_Verb = Request.RequestVerb(); m_ContentLength = Request.Body().Size(); @@ -2083,6 +2102,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode) m_Response.reset(new HttpResponse(HttpContentType::kBinary, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); std::array<IoBuffer, 0> Empty; m_Response->InitializeForPayload((uint16_t)ResponseCode, Empty); @@ -2097,6 +2117,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); m_Response->InitializeForPayload((uint16_t)ResponseCode, Blobs); } @@ -2108,6 +2129,7 @@ HttpAsioServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentT ZEN_ASSERT(!m_Response); m_Response.reset(new HttpResponse(ContentType, m_RequestNumber)); m_Response->SetAllowZeroCopyFileSend(m_AllowZeroCopyFileSend); + m_Response->SetKeepAlive(m_Request.IsKeepAlive()); IoBuffer MessageBuffer(IoBuffer::Wrap, ResponseString.data(), ResponseString.size()); std::array<IoBuffer, 1> SingleBufferList({MessageBuffer}); diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index a02ca7be3..0d8550c5b 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -865,7 +865,7 @@ ZenComputeServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as COMPUTE (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index 696991403..b0ae0a8b1 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -337,7 +337,7 @@ ZenHubServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as HUB (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/proxy/zenproxyserver.cpp b/src/zenserver/proxy/zenproxyserver.cpp index acfdad45f..c768e940a 100644 --- a/src/zenserver/proxy/zenproxyserver.cpp +++ b/src/zenserver/proxy/zenproxyserver.cpp @@ -359,7 +359,7 @@ ZenProxyServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running as PROXY (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/storage/objectstore/objectstore.cpp b/src/zenserver/storage/objectstore/objectstore.cpp index 052c3d630..e347e2dfe 100644 --- a/src/zenserver/storage/objectstore/objectstore.cpp +++ b/src/zenserver/storage/objectstore/objectstore.cpp @@ -271,7 +271,7 @@ HttpObjectStoreService::Inititalize() CreateDirectories(BucketsPath); } - static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]() ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; static constexpr AsciiSet ValidBucketCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789-_.ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; m_Router.AddMatcher("path", @@ -292,10 +292,9 @@ HttpObjectStoreService::Inititalize() m_Router.RegisterRoute( "bucket/{path}", [this](zen::HttpRouterRequest& Request) { - const std::string_view EncodedPath = Request.GetCapture(1); - const std::string Path = Request.ServerRequest().Decode(EncodedPath); - const auto Sep = Path.find_last_of('.'); - const bool IsObject = Sep != std::string::npos && Path.size() - Sep > 0; + const std::string_view Path = Request.GetCapture(1); + const auto Sep = Path.find_last_of('.'); + const bool IsObject = Sep != std::string_view::npos && Path.size() - Sep > 0; if (IsObject) { @@ -378,7 +377,7 @@ HttpObjectStoreService::ListBucket(zen::HttpRouterRequest& Request, const std::s const auto QueryParms = Request.ServerRequest().GetQueryParams(); if (auto PrefixParam = QueryParms.GetValue("prefix"); PrefixParam.empty() == false) { - BucketPrefix = PrefixParam; + BucketPrefix = HttpServerRequest::Decode(PrefixParam); } } BucketPrefix.erase(0, BucketPrefix.find_first_not_of('/')); diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index 77588bd6c..bba5e0a61 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -720,7 +720,7 @@ ZenStorageServer::Run() ExtendableStringBuilder<256> BuildOptions; GetBuildOptions(BuildOptions, '\n'); - ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + ZEN_INFO("Build options ({}/{}, {}):\n{}", GetOperatingSystemName(), GetCpuName(), GetCompilerName(), BuildOptions); } ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 519176ffe..6760e0372 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -201,6 +201,9 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: std::chrono::system_clock::now().time_since_epoch()).count(), .BuildOptions = { {"ZEN_ADDRESS_SANITIZER", ZEN_ADDRESS_SANITIZER != 0}, + {"ZEN_THREAD_SANITIZER", ZEN_THREAD_SANITIZER != 0}, + {"ZEN_MEMORY_SANITIZER", ZEN_MEMORY_SANITIZER != 0}, + {"ZEN_LEAK_SANITIZER", ZEN_LEAK_SANITIZER != 0}, {"ZEN_USE_SENTRY", ZEN_USE_SENTRY != 0}, {"ZEN_WITH_TESTS", ZEN_WITH_TESTS != 0}, {"ZEN_USE_MIMALLOC", ZEN_USE_MIMALLOC != 0}, @@ -251,6 +254,12 @@ ZenServerBase::GetBuildOptions(StringBuilderBase& OutOptions, char Separator) co OutOptions << "ZEN_ADDRESS_SANITIZER=" << (ZEN_ADDRESS_SANITIZER ? "1" : "0"); OutOptions << Separator; + OutOptions << "ZEN_THREAD_SANITIZER=" << (ZEN_THREAD_SANITIZER ? "1" : "0"); + OutOptions << Separator; + OutOptions << "ZEN_MEMORY_SANITIZER=" << (ZEN_MEMORY_SANITIZER ? "1" : "0"); + OutOptions << Separator; + OutOptions << "ZEN_LEAK_SANITIZER=" << (ZEN_LEAK_SANITIZER ? "1" : "0"); + OutOptions << Separator; OutOptions << "ZEN_USE_SENTRY=" << (ZEN_USE_SENTRY ? "1" : "0"); OutOptions << Separator; OutOptions << "ZEN_WITH_TESTS=" << (ZEN_WITH_TESTS ? "1" : "0"); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index b3450b805..f3edf804d 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1776,11 +1776,13 @@ GcScheduler::Initialize(const GcSchedulerConfig& Config) m_LastGcTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcTime"sv].AsInt64())); m_LastGcExpireTime = GcClock::TimePoint(GcClock::Duration(SchedulerState["LastGcExpireTime"].AsInt64(GcClock::Duration::min().count()))); - if (m_LastGcTime + m_Config.Interval < GcClock::Now()) + if (m_LastGcTime > GcClock::Now() || m_LastGcTime + m_Config.Interval < GcClock::Now()) { - // TODO: Trigger GC? + // Reset if the stored timestamp is in the future (e.g. clock resolution mismatch + // between the build that wrote gc_state and this build) or too far in the past. m_LastGcTime = GcClock::Now(); m_LastLightweightGcTime = m_LastGcTime; + m_LastGcExpireTime = GcClock::TimePoint::min(); } m_AttachmentPassIndex = SchedulerState["AttachmentPassIndex"sv].AsUInt8(); } @@ -2084,6 +2086,10 @@ GcScheduler::GetState() const { Result.RemainingTimeUntilFullGc = std::chrono::seconds::zero(); } + else if (Result.RemainingTimeUntilFullGc > Result.Config.Interval) + { + Result.RemainingTimeUntilFullGc = Result.Config.Interval; + } Result.RemainingTimeUntilLightweightGc = Result.Config.LightweightInterval.count() == 0 @@ -2094,6 +2100,10 @@ GcScheduler::GetState() const { Result.RemainingTimeUntilLightweightGc = std::chrono::seconds::zero(); } + else if (Result.RemainingTimeUntilLightweightGc > Result.Config.LightweightInterval) + { + Result.RemainingTimeUntilLightweightGc = Result.Config.LightweightInterval; + } } return Result; @@ -2418,6 +2428,10 @@ GcScheduler::SchedulerThread() { RemainingTimeUntilGc = std::chrono::seconds::zero(); } + else if (RemainingTimeUntilGc > GcInterval) + { + RemainingTimeUntilGc = GcInterval; + } std::chrono::seconds RemainingTimeUntilLightweightGc = LightweightGcInterval.count() == 0 ? std::chrono::seconds::max() @@ -2428,6 +2442,10 @@ GcScheduler::SchedulerThread() { RemainingTimeUntilLightweightGc = std::chrono::seconds::zero(); } + else if (RemainingTimeUntilLightweightGc > LightweightGcInterval) + { + RemainingTimeUntilLightweightGc = LightweightGcInterval; + } // Don't schedule a lightweight GC if a full GC is // due quite soon anyway @@ -20,3 +20,15 @@ # TSAN reports as a race. This is benign: the slot is always NULL and writing NULL # to it has no observable effect. race:eastl::hashtable*DoFreeNodes* + +# UE::Trace's GetUid() uses a racy static uint32 cache (Uid = Uid ? Uid : Initialize()) +# as a fast path to avoid re-entering Initialize(). The actual initialization is done via +# a thread-safe static (Uid_ThreadSafeInit) inside Initialize(), so the worst case is +# redundant calls to Initialize() which always returns the same value. +race:*Fields::GetUid* + +# TRACE_CPU_SCOPE generates a function-local `static int32 scope_id` that is lazily +# initialized without synchronization (if (0 == scope_id) scope_id = ScopeNew(...)). +# Same benign pattern as GetUid: the worst case is redundant calls to ScopeNew() which +# always returns the same value for a given scope name. +race:*$trace_scope_id* @@ -48,12 +48,27 @@ set_policy("build.sanitizer.address", use_asan) -- ThreadSanitizer, MemorySanitizer, LeakSanitizer, and UndefinedBehaviorSanitizer -- are supported on Linux and MacOS only. +-- +-- You can enable these by editing the xmake.lua directly, or by passing the +-- appropriate flags on the command line: +-- +-- `xmake --policies=build.sanitizer.thread:y` for ThreadSanitizer, +-- `xmake --policies=build.sanitizer.memory:y` for MemorySanitizer, etc. ---set_policy("build.sanitizer.thread", true) ---set_policy("build.sanitizer.memory", true) +-- When using TSAN you will want to also use the suppression tile to silence +-- known benign races. You do this by ensuring the the TSAN_OPTIONS environment +-- vriable is set to something like `TSAN_OPTIONS="suppressions=$(projectdir)/tsan.supp"` +-- +-- `prompt> TSAN_OPTIONS="detect_deadlocks=0 suppressions=$(projectdir)/tsan.supp" xmake run zenserver` + +--set_policy("build.sanitizer.thread", true) --set_policy("build.sanitizer.leak", true) --set_policy("build.sanitizer.undefined", true) +-- In practice, this does not work because of the difficulty of compiling +-- dependencies with MemorySanitizer. +--set_policy("build.sanitizer.memory", true) + -------------------------------------------------------------------------- -- Dependencies @@ -470,401 +485,6 @@ task("test") } } on_run(function() - import("core.base.option") - import("core.project.config") - import("core.project.project") - - config.load() - - -- Override table: target name -> short name (for targets that don't follow convention) - local short_name_overrides = { - ["zenserver-test"] = "integration", - } - - -- Build test list from targets in the "tests" group - local available_tests = {} - for name, target in pairs(project.targets()) do - if target:get("group") == "tests" and name:endswith("-test") then - local short = short_name_overrides[name] - if not short then - -- Derive short name: "zencore-test" -> "core" - short = name - if short:startswith("zen") then short = short:sub(4) end - if short:endswith("-test") then short = short:sub(1, -6) end - end - table.insert(available_tests, {short, name}) - end - end - - -- Add non-test-group entries that have a test subcommand - table.insert(available_tests, {"server", "zenserver"}) - - table.sort(available_tests, function(a, b) return a[1] < b[1] end) - - -- Handle --list: print discovered test names and exit - if option.get("list") then - printf("Available tests:\n") - for _, entry in ipairs(available_tests) do - printf(" %-16s -> %s\n", entry[1], entry[2]) - end - return - end - - local testname = option.get("run") - - -- Parse comma-separated test names into a set - local requested = {} - for token in testname:gmatch("[^,]+") do - requested[token:match("^%s*(.-)%s*$")] = true - end - - -- Filter to requested test(s) - local tests = {} - local matched = {} - - for _, entry in ipairs(available_tests) do - local name, target = entry[1], entry[2] - if requested["all"] or requested[name] then - table.insert(tests, {name = name, target = target}) - matched[name] = true - end - end - - -- Check for unknown test names - if not requested["all"] then - for name, _ in pairs(requested) do - if not matched[name] then - raise("no tests match specification: '%s'", name) - end - end - end - - if #tests == 0 then - raise("no tests match specification: '%s'", testname) - end - - local plat, arch - if is_host("windows") then - plat = "windows" - arch = "x64" - elseif is_host("macosx") then - plat = "macosx" - arch = is_arch("arm64") and "arm64" or "x86_64" - else - plat = "linux" - arch = "x86_64" - end - - -- Only reconfigure if current config doesn't already match - if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then - local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or "" - local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or "" - os.exec("xmake config -y -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag) - end - - -- Build targets we're going to run - if requested["all"] then - os.exec("xmake build -y") - else - for _, entry in ipairs(tests) do - os.exec("xmake build -y %s", entry.target) - end - end - - local use_junit_reporting = option.get("junit") - local use_noskip = option.get("noskip") - local use_verbose = option.get("verbose") - local repeat_count = tonumber(option.get("repeat")) or 1 - local extra_args = option.get("arguments") or {} - local junit_report_files = {} - - local junit_report_dir - if use_junit_reporting then - junit_report_dir = path.join(os.projectdir(), config.get("buildir"), "reports") - os.mkdir(junit_report_dir) - end - - -- Results collection for summary table - local results = {} - local any_failed = false - - -- Format a number with thousands separators (e.g. 31103 -> "31,103") - local function format_number(n) - local s = tostring(n) - local pos = #s % 3 - if pos == 0 then pos = 3 end - local result = s:sub(1, pos) - for i = pos + 1, #s, 3 do - result = result .. "," .. s:sub(i, i + 2) - end - return result - end - - -- Center a string within a given width - local function center_str(s, width) - local pad = width - #s - local lpad = math.floor(pad / 2) - local rpad = pad - lpad - return string.rep(" ", lpad) .. s .. string.rep(" ", rpad) - end - - -- Left-align a string within a given width (with 1-space left margin) - local function left_align_str(s, width) - return " " .. s .. string.rep(" ", width - #s - 1) - end - - -- Right-align a string within a given width (with 1-space right margin) - local function right_align_str(s, width) - return string.rep(" ", width - #s - 1) .. s .. " " - end - - -- Format elapsed seconds as a human-readable string - local function format_time(seconds) - if seconds >= 60 then - local mins = math.floor(seconds / 60) - local secs = seconds - mins * 60 - return string.format("%dm %04.1fs", mins, secs) - else - return string.format("%.1fs", seconds) - end - end - - -- Parse test summary file written by TestListener - local function parse_summary_file(filepath) - if not os.isfile(filepath) then return nil end - local content = io.readfile(filepath) - if not content then return nil end - local ct = content:match("cases_total=(%d+)") - local cp = content:match("cases_passed=(%d+)") - local at = content:match("assertions_total=(%d+)") - local ap = content:match("assertions_passed=(%d+)") - if ct then - local failures = {} - for name, file, line in content:gmatch("failed=([^|\n]+)|([^|\n]+)|(%d+)") do - table.insert(failures, {name = name, file = file, line = tonumber(line)}) - end - local es = content:match("elapsed_seconds=([%d%.]+)") - return { - cases_total = tonumber(ct), - cases_passed = tonumber(cp) or 0, - asserts_total = tonumber(at) or 0, - asserts_passed = tonumber(ap) or 0, - elapsed_seconds = tonumber(es) or 0, - failures = failures - } - end - return nil - end - - -- Temp directory for summary files - local summary_dir = path.join(os.tmpdir(), "zen-test-summary") - os.mkdir(summary_dir) - - -- Run each test suite and collect results - for iteration = 1, repeat_count do - if repeat_count > 1 then - printf("\n*** Iteration %d/%d ***\n", iteration, repeat_count) - end - - for _, entry in ipairs(tests) do - local name, target = entry.name, entry.target - printf("=== %s ===\n", target) - - local suite_name = target - if name == "server" then - suite_name = "zenserver (test)" - end - - local cmd = string.format("xmake run %s", target) - if name == "server" then - cmd = string.format("xmake run %s test", target) - end - cmd = string.format("%s --duration=true", cmd) - - if use_junit_reporting then - local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target)) - junit_report_files[target] = junit_report_file - cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) - end - if use_noskip then - cmd = string.format("%s --no-skip", cmd) - end - if use_verbose and name == "integration" then - cmd = string.format("%s --verbose", cmd) - end - for _, arg in ipairs(extra_args) do - cmd = string.format("%s %s", cmd, arg) - end - - -- Tell TestListener where to write the summary - local summary_file = path.join(summary_dir, target .. ".txt") - os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file) - - -- Run test with real-time streaming output - local test_ok = true - try { - function() - os.exec(cmd) - end, - catch { - function(errors) - test_ok = false - end - } - } - - -- Read summary written by TestListener - local summary = parse_summary_file(summary_file) - os.tryrm(summary_file) - - if not test_ok then - any_failed = true - end - - table.insert(results, { - suite = suite_name, - cases_passed = summary and summary.cases_passed or 0, - cases_total = summary and summary.cases_total or 0, - asserts_passed = summary and summary.asserts_passed or 0, - asserts_total = summary and summary.asserts_total or 0, - elapsed_seconds = summary and summary.elapsed_seconds or 0, - failures = summary and summary.failures or {}, - passed = test_ok - }) - end - - if any_failed then - if repeat_count > 1 then - printf("\n*** Failure detected on iteration %d, stopping ***\n", iteration) - end - break - end - end - - -- Clean up - os.setenv("ZEN_TEST_SUMMARY_FILE", "") - os.tryrm(summary_dir) - - -- Print JUnit reports if requested - for test, junit_report_file in pairs(junit_report_files) do - printf("=== report - %s ===\n", test) - if os.isfile(junit_report_file) then - local data = io.readfile(junit_report_file) - if data then - print(data) - end - end - end - - -- Print summary table - if #results > 0 then - -- Calculate column widths based on content - local col_suite = #("Suite") - local col_cases = #("Cases") - local col_asserts = #("Assertions") - local col_time = #("Time") - local col_status = #("Status") - - -- Compute totals - local total_cases_passed = 0 - local total_cases_total = 0 - local total_asserts_passed = 0 - local total_asserts_total = 0 - local total_elapsed = 0 - - for _, r in ipairs(results) do - col_suite = math.max(col_suite, #r.suite) - local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) - col_cases = math.max(col_cases, #cases_str) - local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) - col_asserts = math.max(col_asserts, #asserts_str) - col_time = math.max(col_time, #format_time(r.elapsed_seconds)) - local status_str = r.passed and "SUCCESS" or "FAILED" - col_status = math.max(col_status, #status_str) - - total_cases_passed = total_cases_passed + r.cases_passed - total_cases_total = total_cases_total + r.cases_total - total_asserts_passed = total_asserts_passed + r.asserts_passed - total_asserts_total = total_asserts_total + r.asserts_total - total_elapsed = total_elapsed + r.elapsed_seconds - end - - -- Account for totals row in column widths - col_suite = math.max(col_suite, #("Total")) - col_cases = math.max(col_cases, #(format_number(total_cases_passed) .. "/" .. format_number(total_cases_total))) - col_asserts = math.max(col_asserts, #(format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total))) - col_time = math.max(col_time, #format_time(total_elapsed)) - - -- Add padding (1 space each side) - col_suite = col_suite + 2 - col_cases = col_cases + 2 - col_asserts = col_asserts + 2 - col_time = col_time + 2 - col_status = col_status + 2 - - -- Build horizontal border segments - local h_suite = string.rep("-", col_suite) - local h_cases = string.rep("-", col_cases) - local h_asserts = string.rep("-", col_asserts) - local h_time = string.rep("-", col_time) - local h_status = string.rep("-", col_status) - - local top = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" - local mid = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" - local bottom = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" - local vbar = "|" - - local header_msg = any_failed and "Some tests failed:" or "All tests passed:" - printf("\n* %s\n", header_msg) - printf(" %s\n", top) - printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, center_str("Suite", col_suite), vbar, center_str("Cases", col_cases), vbar, center_str("Assertions", col_asserts), vbar, center_str("Time", col_time), vbar, center_str("Status", col_status), vbar) - - for _, r in ipairs(results) do - printf(" %s\n", mid) - local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) - local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) - local time_str = format_time(r.elapsed_seconds) - local status_str = r.passed and "SUCCESS" or "FAILED" - printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str(r.suite, col_suite), vbar, right_align_str(cases_str, col_cases), vbar, right_align_str(asserts_str, col_asserts), vbar, right_align_str(time_str, col_time), vbar, right_align_str(status_str, col_status), vbar) - end - - -- Totals row - if #results > 1 then - local h_suite_eq = string.rep("=", col_suite) - local h_cases_eq = string.rep("=", col_cases) - local h_asserts_eq = string.rep("=", col_asserts) - local h_time_eq = string.rep("=", col_time) - local h_status_eq = string.rep("=", col_status) - local totals_sep = "+" .. h_suite_eq .. "+" .. h_cases_eq .. "+" .. h_asserts_eq .. "+" .. h_time_eq .. "+" .. h_status_eq .. "+" - printf(" %s\n", totals_sep) - - local total_cases_str = format_number(total_cases_passed) .. "/" .. format_number(total_cases_total) - local total_asserts_str = format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total) - local total_time_str = format_time(total_elapsed) - local total_status_str = any_failed and "FAILED" or "SUCCESS" - printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str("Total", col_suite), vbar, right_align_str(total_cases_str, col_cases), vbar, right_align_str(total_asserts_str, col_asserts), vbar, right_align_str(total_time_str, col_time), vbar, right_align_str(total_status_str, col_status), vbar) - end - - printf(" %s\n", bottom) - end - - -- Print list of individual failing tests - if any_failed then - printf("\n Failures:\n") - for _, r in ipairs(results) do - if #r.failures > 0 then - printf(" -- %s --\n", r.suite) - for _, f in ipairs(r.failures) do - printf(" FAILED: %s (%s:%d)\n", f.name, f.file, f.line) - end - elseif not r.passed then - printf(" -- %s --\n", r.suite) - printf(" (test binary exited with error, no failure details available)\n") - end - end - end - - if any_failed then - raise("one or more test suites failed") - end + import("scripts.test") + test() end) |